Embed runtime version as a custom section (paritytech#8688)

* emit a custom section from impl_runtime_apis!

This change emits a custom section from the impl_runtime_apis! proc macro.

Each implemented API will result to emitting a link section `runtime_apis`.
During linking all sections with this name will be concatenated and
placed into the final wasm binary under the same name.

* Introduce `runtime_version` proc macro

This macro takes an existing `RuntimeVersion` const declaration, parses
it and emits the version information in form of a linking section.
Ultimately such a linking section will result into a custom wasm
section.

* Parse custom wasm section for runtime version

* Apply suggestions from code review

Co-authored-by: David <[email protected]>

* Fix sc-executor integration tests

* Nits

Co-authored-by: Bastian Köcher <[email protected]>

* Refactor apis section deserialization

* Fix version decoding

* Reuse uncompressed value for CallInWasm

* Log on decompression error

* Simplify if

* Reexport proc-macro from sp_version

* Merge ReadRuntimeVersionExt

* Export `read_embedded_version`

* Fix test

* Simplify searching for custom section

Co-authored-by: David <[email protected]>
Co-authored-by: Bastian Köcher <[email protected]>

Cargo.toml

@@ -191,6 +191,7 @@ members = [
 	"primitives/trie",
 	"primitives/utils",
 	"primitives/version",
+	"primitives/version/proc-macro",
 	"primitives/wasm-interface",
 	"test-utils/client",
 	"test-utils/derive",

bin/node-template/runtime/src/lib.rs

@@ -94,6 +94,7 @@ pub mod opaque {
 
 // To learn more about runtime versioning and what each of the following value means:
 //   https://substrate.dev/docs/en/knowledgebase/runtime/upgrades#runtime-versioning
+#[sp_version::runtime_version]
 pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_name: create_runtime_str!("node-template"),
 	impl_name: create_runtime_str!("node-template"),

bin/node/runtime/src/lib.rs

@@ -105,6 +105,7 @@ pub fn wasm_binary_unwrap() -> &'static [u8] {
 }
 
 /// Runtime version.
+#[sp_version::runtime_version]
 pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_name: create_runtime_str!("node"),
 	impl_name: create_runtime_str!("substrate-node"),

client/executor/Cargo.toml

@@ -30,7 +30,6 @@ sp-api = { version = "3.0.0", path = "../../primitives/api" }
 sp-wasm-interface = { version = "3.0.0", path = "../../primitives/wasm-interface" }
 sp-runtime-interface = { version = "3.0.0", path = "../../primitives/runtime-interface" }
 sp-externalities = { version = "0.9.0", path = "../../primitives/externalities" }
-sp-maybe-compressed-blob = { version = "3.0.0", path = "../../primitives/maybe-compressed-blob" }
 sc-executor-common = { version = "0.9.0", path = "common" }
 sc-executor-wasmi = { version = "0.9.0", path = "wasmi" }
 sc-executor-wasmtime = { version = "0.9.0", path = "wasmtime", optional = true }

client/executor/common/Cargo.toml

@@ -22,6 +22,7 @@ wasmi = "0.6.2"
 sp-core = { version = "3.0.0", path = "../../../primitives/core" }
 sp-allocator = { version = "3.0.0", path = "../../../primitives/allocator" }
 sp-wasm-interface = { version = "3.0.0", path = "../../../primitives/wasm-interface" }
+sp-maybe-compressed-blob = { version = "3.0.0", path = "../../../primitives/maybe-compressed-blob" }
 sp-serializer = { version = "3.0.0", path = "../../../primitives/serializer" }
 thiserror = "1.0.21"
 

client/executor/common/src/runtime_blob/runtime_blob.rs

@@ -27,6 +27,17 @@ pub struct RuntimeBlob {
 }
 
 impl RuntimeBlob {
+	/// Create `RuntimeBlob` from the given wasm code. Will attempt to decompress the code before
+	/// deserializing it.
+	///
+	/// See [`sp_maybe_compressed_blob`] for details about decompression.
+	pub fn uncompress_if_needed(wasm_code: &[u8]) -> Result<Self, WasmError> {
+		use sp_maybe_compressed_blob::CODE_BLOB_BOMB_LIMIT;
+		let wasm_code = sp_maybe_compressed_blob::decompress(wasm_code, CODE_BLOB_BOMB_LIMIT)
+			.map_err(|e| WasmError::Other(format!("Decompression error: {:?}", e)))?;
+		Self::new(&wasm_code)
+	}
+
 	/// Create `RuntimeBlob` from the given wasm code.
 	///
 	/// Returns `Err` if the wasm code cannot be deserialized.
@@ -85,9 +96,23 @@ impl RuntimeBlob {
 		})
 	}
 
+	/// Scans the wasm blob for the first section with the name that matches the given. Returns the
+	/// contents of the custom section if found or `None` otherwise.
+	pub fn custom_section_contents(&self, section_name: &str) -> Option<&[u8]> {
+		self.raw_module
+			.custom_sections()
+			.find(|cs| cs.name() == section_name)
+			.map(|cs| cs.payload())
+		}
+
 	/// Consumes this runtime blob and serializes it.
 	pub fn serialize(self) -> Vec<u8> {
 		serialize(self.raw_module)
 			.expect("serializing into a vec should succeed; qed")
 	}
+
+	/// Destructure this structure into the underlying parity-wasm Module.
+	pub fn into_inner(self) -> RawModule {
+		self.raw_module
+	}
 }

client/executor/src/integration_tests/mod.rs

@@ -17,18 +17,20 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 mod sandbox;
 
+use std::sync::Arc;
 use codec::{Encode, Decode};
 use hex_literal::hex;
 use sp_core::{
 	blake2_128, blake2_256, ed25519, sr25519, map, Pair,
 	offchain::{OffchainWorkerExt, OffchainDbExt, testing},
-	traits::{Externalities, CallInWasm},
+	traits::Externalities,
 };
 use sc_runtime_test::wasm_binary_unwrap;
 use sp_state_machine::TestExternalities as CoreTestExternalities;
 use sp_trie::{TrieConfiguration, trie_types::Layout};
 use sp_wasm_interface::HostFunctions as _;
 use sp_runtime::traits::BlakeTwo256;
+use sc_executor_common::{wasm_runtime::WasmModule, runtime_blob::RuntimeBlob};
 use tracing_subscriber::layer::SubscriberExt;
 
 use crate::WasmExecutionMethod;
@@ -77,13 +79,12 @@ fn call_in_wasm<E: Externalities>(
 		8,
 		None,
 	);
-	executor.call_in_wasm(
-		&wasm_binary_unwrap()[..],
-		None,
+	executor.uncached_call(
+		RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
+		ext,
+		true,
 		function,
 		call_data,
-		ext,
-		sp_core::traits::MissingHostFunctions::Allow,
 	)
 }
 
@@ -541,28 +542,37 @@ fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) {
 		None,
 	);
 
-	let err = executor.call_in_wasm(
-		&wasm_binary_unwrap()[..],
-		None,
-		"test_exhaust_heap",
-		&[0],
-		&mut ext.ext(),
-		sp_core::traits::MissingHostFunctions::Allow,
-	).unwrap_err();
+	let err = executor
+		.uncached_call(
+			RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
+			&mut ext.ext(),
+			true,
+			"test_exhaust_heap",
+			&[0],
+		)
+		.unwrap_err();
 
 	assert!(err.contains("Allocator ran out of space"));
 }
 
-test_wasm_execution!(returns_mutable_static);
-fn returns_mutable_static(wasm_method: WasmExecutionMethod) {
-	let runtime = crate::wasm_runtime::create_wasm_runtime_with_code(
+fn mk_test_runtime(wasm_method: WasmExecutionMethod, pages: u64) -> Arc<dyn WasmModule> {
+	let blob = RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..])
+		.expect("failed to create a runtime blob out of test runtime");
+
+	crate::wasm_runtime::create_wasm_runtime_with_code(
 		wasm_method,
-		1024,
-		&wasm_binary_unwrap()[..],
+		pages,
+		blob,
 		HostFunctions::host_functions(),
 		true,
 		None,
-	).expect("Creates runtime");
+	)
+	.expect("failed to instantiate wasm runtime")
+}
+
+test_wasm_execution!(returns_mutable_static);
+fn returns_mutable_static(wasm_method: WasmExecutionMethod) {
+	let runtime = mk_test_runtime(wasm_method, 1024);
 
 	let instance = runtime.new_instance().unwrap();
 	let res = instance.call_export("returns_mutable_static", &[0]).unwrap();
@@ -589,14 +599,7 @@ fn restoration_of_globals(wasm_method: WasmExecutionMethod) {
 	// to our allocator algorithm there are inefficiencies.
 	const REQUIRED_MEMORY_PAGES: u64 = 32;
 
-	let runtime = crate::wasm_runtime::create_wasm_runtime_with_code(
-		wasm_method,
-		REQUIRED_MEMORY_PAGES,
-		&wasm_binary_unwrap()[..],
-		HostFunctions::host_functions(),
-		true,
-		None,
-	).expect("Creates runtime");
+	let runtime = mk_test_runtime(wasm_method, REQUIRED_MEMORY_PAGES);
 	let instance = runtime.new_instance().unwrap();
 
 	// On the first invocation we allocate approx. 768KB (75%) of stack and then trap.
@@ -610,14 +613,7 @@ fn restoration_of_globals(wasm_method: WasmExecutionMethod) {
 
 test_wasm_execution!(interpreted_only heap_is_reset_between_calls);
 fn heap_is_reset_between_calls(wasm_method: WasmExecutionMethod) {
-	let runtime = crate::wasm_runtime::create_wasm_runtime_with_code(
-		wasm_method,
-		1024,
-		&wasm_binary_unwrap()[..],
-		HostFunctions::host_functions(),
-		true,
-		None,
-	).expect("Creates runtime");
+	let runtime = mk_test_runtime(wasm_method, 1024);
 	let instance = runtime.new_instance().unwrap();
 
 	let heap_base = instance.get_global_const("__heap_base")
@@ -642,27 +638,27 @@ fn parallel_execution(wasm_method: WasmExecutionMethod) {
 		8,
 		None,
 	));
-	let code_hash = blake2_256(wasm_binary_unwrap()).to_vec();
-	let threads: Vec<_> = (0..8).map(|_|
-		{
+	let threads: Vec<_> = (0..8)
+		.map(|_| {
 			let executor = executor.clone();
-			let code_hash = code_hash.clone();
 			std::thread::spawn(move || {
 				let mut ext = TestExternalities::default();
 				let mut ext = ext.ext();
 				assert_eq!(
-					executor.call_in_wasm(
-						&wasm_binary_unwrap()[..],
-						Some(code_hash.clone()),
-						"test_twox_128",
-						&[0],
-						&mut ext,
-						sp_core::traits::MissingHostFunctions::Allow,
-					).unwrap(),
+					executor
+						.uncached_call(
+							RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
+							&mut ext,
+							true,
+							"test_twox_128",
+							&[0],
+						)
+						.unwrap(),
 					hex!("99e9d85137db46ef4bbea33613baafd5").to_vec().encode(),
 				);
 			})
-		}).collect();
+		})
+		.collect();
 
 	for t in threads.into_iter() {
 		t.join().unwrap();
@@ -671,9 +667,7 @@ fn parallel_execution(wasm_method: WasmExecutionMethod) {
 
 test_wasm_execution!(wasm_tracing_should_work);
 fn wasm_tracing_should_work(wasm_method: WasmExecutionMethod) {
-
-	use std::sync::{Arc, Mutex};
-
+	use std::sync::Mutex;
 	use sc_tracing::{SpanDatum, TraceEvent};
 
 	struct TestTraceHandler(Arc<Mutex<Vec<SpanDatum>>>);
@@ -779,6 +773,5 @@ fn panic_in_spawned_instance_panics_on_joining_its_result(wasm_method: WasmExecu
 		&mut ext,
 	).unwrap_err();
 
-	dbg!(&error_result);
 	assert!(format!("{}", error_result).contains("Spawned task"));
 }

client/executor/src/lib.rs

@@ -38,14 +38,17 @@ mod wasm_runtime;
 mod integration_tests;
 
 pub use wasmi;
-pub use native_executor::{with_externalities_safe, NativeExecutor, WasmExecutor, NativeExecutionDispatch};
+pub use native_executor::{
+	with_externalities_safe, NativeExecutor, WasmExecutor, NativeExecutionDispatch,
+};
 pub use sp_version::{RuntimeVersion, NativeVersion};
 pub use codec::Codec;
 #[doc(hidden)]
-pub use sp_core::traits::{Externalities, CallInWasm};
+pub use sp_core::traits::{Externalities};
 #[doc(hidden)]
 pub use sp_wasm_interface;
 pub use wasm_runtime::WasmExecutionMethod;
+pub use wasm_runtime::read_embedded_version;
 
 pub use sc_executor_common::{error, sandbox};
 
@@ -68,7 +71,7 @@ mod tests {
 	use sc_runtime_test::wasm_binary_unwrap;
 	use sp_io::TestExternalities;
 	use sp_wasm_interface::HostFunctions;
-	use sp_core::traits::CallInWasm;
+	use sc_executor_common::runtime_blob::RuntimeBlob;
 
 	#[test]
 	fn call_in_interpreted_wasm_works() {
@@ -82,14 +85,15 @@ mod tests {
 			8,
 			None,
 		);
-		let res = executor.call_in_wasm(
-			&wasm_binary_unwrap()[..],
-			None,
-			"test_empty_return",
-			&[],
-			&mut ext,
-			sp_core::traits::MissingHostFunctions::Allow,
-		).unwrap();
+		let res = executor
+			.uncached_call(
+				RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
+				&mut ext,
+				true,
+				"test_empty_return",
+				&[],
+			)
+			.unwrap();
 		assert_eq!(res, vec![0u8; 0]);
 	}
 }