diff --git a/Cargo.lock b/Cargo.lock index a33cb02f7f0d4..4d5d812895321 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7469,7 +7469,6 @@ dependencies = [ "parity-wasm 0.42.2", "parking_lot 0.11.1", "paste 1.0.4", - "regex", "sc-executor-common", "sc-executor-wasmi", "sc-executor-wasmtime", @@ -7501,8 +7500,10 @@ name = "sc-executor-common" version = "0.9.0" dependencies = [ "derive_more", + "mach", "parity-scale-codec", "pwasm-utils", + "regex", "sp-allocator", "sp-core", "sp-maybe-compressed-blob", @@ -7531,9 +7532,11 @@ name = "sc-executor-wasmtime" version = "0.9.0" dependencies = [ "assert_matches", + "bitflags", "cfg-if 1.0.0", "libc", "log", + "mach", "parity-scale-codec", "parity-wasm 0.42.2", "sc-executor-common", diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml index 27e90ddcc85e6..7cb2e12fd3913 100644 --- a/client/executor/Cargo.toml +++ b/client/executor/Cargo.toml @@ -50,7 +50,6 @@ sc-tracing = { version = "3.0.0", path = "../tracing" } tracing = "0.1.25" tracing-subscriber = "0.2.18" paste = "1.0" -regex = "1" [features] default = [ "std" ] diff --git a/client/executor/common/Cargo.toml b/client/executor/common/Cargo.toml index cb238f3a96fb0..0a9b3ad365cbe 100644 --- a/client/executor/common/Cargo.toml +++ b/client/executor/common/Cargo.toml @@ -13,6 +13,12 @@ readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +[target.'cfg(target_os = "macos")'.dependencies] +mach = "0.3" + +[target.'cfg(target_os = "linux")'.dependencies] +regex = "1" + [dependencies] derive_more = "0.99.2" pwasm-utils = "0.18.0" diff --git a/client/executor/common/src/lib.rs b/client/executor/common/src/lib.rs index 25e06314aba39..f002514d6b35f 100644 --- a/client/executor/common/src/lib.rs +++ b/client/executor/common/src/lib.rs @@ -25,3 +25,4 @@ pub mod error; pub mod sandbox; pub mod wasm_runtime; pub mod runtime_blob; +pub mod test_utils; diff --git a/client/executor/common/src/test_utils.rs b/client/executor/common/src/test_utils.rs new file mode 100644 index 0000000000000..f0cc495a58eeb --- /dev/null +++ b/client/executor/common/src/test_utils.rs @@ -0,0 +1,32 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Helper functions that are useful during testing. Those are mostly operating system +//! dependend because they are used to inspect the current state of the system. + +#[cfg(target_os = "linux")] +mod linux; + +#[cfg(target_os = "linux")] +pub use linux::*; + +#[cfg(target_os = "macos")] +mod macos; + +#[cfg(target_os = "macos")] +pub use macos::*; diff --git a/client/executor/src/integration_tests/linux/smaps.rs b/client/executor/common/src/test_utils/linux.rs similarity index 77% rename from client/executor/src/integration_tests/linux/smaps.rs rename to client/executor/common/src/test_utils/linux.rs index 8088a5a3ea952..d9d8746d87d7c 100644 --- a/client/executor/src/integration_tests/linux/smaps.rs +++ b/client/executor/common/src/test_utils/linux.rs @@ -16,11 +16,19 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! A tool for extracting information about the memory consumption of the current process from -//! the procfs. +//! Implementation of Linux specific tests and/or helper functions. -use std::ops::Range; -use std::collections::BTreeMap; +use crate::wasm_runtime::WasmInstance; +use std::{ + ops::Range, + collections::BTreeMap, +}; + +/// Returns how much bytes of the instance's memory is currently resident (backed by phys mem) +pub fn instance_resident_bytes(instance: &dyn WasmInstance) -> usize { + let base_addr = instance.linear_memory_range().unwrap().start; + Smaps::new().get_rss(base_addr).expect("failed to get rss") +} /// An interface to the /proc/self/smaps /// @@ -30,6 +38,7 @@ use std::collections::BTreeMap; pub struct Smaps(Vec<(Range, BTreeMap)>); impl Smaps { + /// Create a in-memory representation of the calling processe's memory map. pub fn new() -> Self { let regex_start = regex::RegexBuilder::new("^([0-9a-f]+)-([0-9a-f]+)") .multi_line(true) @@ -68,6 +77,13 @@ impl Smaps { Self(output) } + /// Returns how much memory is currently resident in the memory mapping that is + /// associated with the specified address. + pub fn get_rss(&self, addr: usize) -> Option { + self.get_map(addr).get("Rss").cloned() + } + + /// Get the mapping at the specified address. fn get_map(&self, addr: usize) -> &BTreeMap { &self.0 .iter() @@ -75,8 +91,4 @@ impl Smaps { .unwrap() .1 } - - pub fn get_rss(&self, addr: usize) -> Option { - self.get_map(addr).get("Rss").cloned() - } } diff --git a/client/executor/common/src/test_utils/macos.rs b/client/executor/common/src/test_utils/macos.rs new file mode 100644 index 0000000000000..13fa7fa3ee3ee --- /dev/null +++ b/client/executor/common/src/test_utils/macos.rs @@ -0,0 +1,109 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Implementation of macOS specific tests and/or helper functions. + +use crate::wasm_runtime::WasmInstance; +use std::{convert::TryInto, mem::MaybeUninit, ops::Range, fmt}; +use mach::{ + kern_return::KERN_SUCCESS, + traps::mach_task_self, + vm::mach_vm_region, + vm_page_size::vm_page_shift, + vm_region::{vm_region_extended_info, vm_region_info_t, VM_REGION_EXTENDED_INFO}, +}; + +/// Size and metadata of a memory mapped region. +pub struct Region { + /// The virtual memory range (addr..addr + size) of the region. + pub range: Range, + /// Metadata describing the memory mapping. + pub info: vm_region_extended_info, +} + +impl fmt::Display for Region { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{:016x?}: {:#?}", self.range, self.info) + } +} + +impl Region { + /// The length of the covered area in bytes. + pub fn len(&self) -> u64 { + self.range.end - self.range.start + } +} + +/// Returns how much bytes of the instance's memory is currently resident (backed by phys mem) +pub fn instance_resident_bytes(instance: &dyn WasmInstance) -> usize { + let range = instance.linear_memory_range().unwrap(); + let regions = get_regions((range.start as u64)..(range.end as u64)).unwrap(); + assert_ne!(regions.len(), 0); + let resident_pages: u64 = regions.iter().map(|r| u64::from(r.info.pages_resident)).sum(); + let resident_size = unsafe { resident_pages << vm_page_shift }; + resident_size.try_into().unwrap() +} + +/// Get all consecutive memory mappings that lie inside the specified range. +/// +/// Returns an error if some parts of the range are unmapped. +pub fn get_regions(range: Range) -> Result, String> { + let mut regions = Vec::new(); + let mut addr = range.start; + + loop { + let mut size = MaybeUninit::::uninit(); + let mut info = MaybeUninit::::uninit(); + let result = unsafe { + mach_vm_region( + mach_task_self(), + &mut addr, + size.as_mut_ptr(), + VM_REGION_EXTENDED_INFO, + (info.as_mut_ptr()) as vm_region_info_t, + &mut vm_region_extended_info::count(), + &mut 0, + ) + }; + assert_eq!(result, KERN_SUCCESS, "mach_vm_region returned an error"); + if result != KERN_SUCCESS { + Err(format!("Failed to get region at address 0x{:016x} with error {}", addr, result))?; + } + + let size = unsafe { size.assume_init() }; + let info = unsafe { info.assume_init() }; + + // We only consider mappings that are fully enclosed by the supplied range + if addr < range.start || addr + size > range.end { + break; + } + + regions.push(Region { + range: addr..(addr + size), + info, + }); + + // Check whether this is the last region. + addr += size; + if addr == range.end { + break; + } + } + + Ok(regions) +} diff --git a/client/executor/common/src/wasm_runtime.rs b/client/executor/common/src/wasm_runtime.rs index 12ff92a2c607f..b20b0ed1fe8a0 100644 --- a/client/executor/common/src/wasm_runtime.rs +++ b/client/executor/common/src/wasm_runtime.rs @@ -20,6 +20,7 @@ use crate::error::Error; use sp_wasm_interface::Value; +use std::ops::Range; /// A method to be used to find the entrypoint when calling into the runtime /// @@ -94,12 +95,14 @@ pub trait WasmInstance: Send { /// This method is only suitable for getting immutable globals. fn get_global_const(&self, name: &str) -> Result, Error>; - /// **Testing Only**. This function returns the base address of the linear memory. + /// **Testing Only**. This function returns the memory range covered by linear memory. /// - /// This is meant to be the starting address of the memory mapped area for the linear memory. + /// The start of the range is the starting address of the memory mapped area for the + /// linear memory as host memory virtual address. /// - /// This function is intended only for a specific test that measures physical memory consumption. - fn linear_memory_base_ptr(&self) -> Option<*const u8> { + /// This function is intended only for a specific test that measures physical memory + /// consumption. + fn linear_memory_range(&self) -> Option> { None } } diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 0762306309df4..b733a1e7abe93 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -16,8 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -#[cfg(target_os = "linux")] -mod linux; +mod resident_memory; mod sandbox; use std::sync::Arc; diff --git a/client/executor/src/integration_tests/linux.rs b/client/executor/src/integration_tests/resident_memory.rs similarity index 80% rename from client/executor/src/integration_tests/linux.rs rename to client/executor/src/integration_tests/resident_memory.rs index 057cc1332717b..dd5f645aad239 100644 --- a/client/executor/src/integration_tests/linux.rs +++ b/client/executor/src/integration_tests/resident_memory.rs @@ -16,20 +16,22 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Tests that are only relevant for Linux. +//! Tests that that resident memory is handled correctly. // Constrain this only to wasmtime for the time being. Without this rustc will complain on unused // imports and items. The alternative is to plop `cfg(feature = wasmtime)` everywhere which seems -// borthersome. +// bothersome. #![cfg(feature = "wasmtime")] -use crate::WasmExecutionMethod; -use super::mk_test_runtime; -use codec::Encode as _; - -mod smaps; +// As of right now we don't have windows support for inspecting resident memory. +#![cfg(any(target_os = "linux", target_os = "macos"))] -use self::smaps::Smaps; +use crate::{ + WasmExecutionMethod, + integration_tests::mk_test_runtime, +}; +use codec::Encode as _; +use sc_executor_common::test_utils::instance_resident_bytes; #[test] fn memory_consumption_compiled() { @@ -48,25 +50,20 @@ fn memory_consumption_compiled() { .as_i32() .expect("`__heap_base` is an `i32`"); - fn probe_rss(instance: &dyn sc_executor_common::wasm_runtime::WasmInstance) -> usize { - let base_addr = instance.linear_memory_base_ptr().unwrap() as usize; - Smaps::new().get_rss(base_addr).expect("failed to get rss") - } - instance .call_export( "test_dirty_plenty_memory", &(heap_base as u32, 1u32).encode(), ) .unwrap(); - let probe_1 = probe_rss(&*instance); + let probe_1 = instance_resident_bytes(&*instance); instance .call_export( "test_dirty_plenty_memory", &(heap_base as u32, 1024u32).encode(), ) .unwrap(); - let probe_2 = probe_rss(&*instance); + let probe_2 = instance_resident_bytes(&*instance); assert_eq!(probe_1, 0); assert_eq!(probe_2, 0); diff --git a/client/executor/wasmtime/Cargo.toml b/client/executor/wasmtime/Cargo.toml index 1e886d15beb18..dd2f8c976253d 100644 --- a/client/executor/wasmtime/Cargo.toml +++ b/client/executor/wasmtime/Cargo.toml @@ -12,6 +12,10 @@ readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +[target.'cfg(target_os = "macos")'.dependencies] +mach = "0.3" +bitflags = "1" + [dependencies] libc = "0.2.90" cfg-if = "1.0" diff --git a/client/executor/wasmtime/src/instance_wrapper.rs b/client/executor/wasmtime/src/instance_wrapper.rs index 866dbfb2e2bfc..7658460ef705a 100644 --- a/client/executor/wasmtime/src/instance_wrapper.rs +++ b/client/executor/wasmtime/src/instance_wrapper.rs @@ -22,7 +22,7 @@ use crate::util; use crate::imports::Imports; -use std::{slice, marker}; +use std::{slice, marker, ops::Range}; use sc_executor_common::{ error::{Error, Result}, runtime_blob, @@ -416,9 +416,11 @@ impl InstanceWrapper { } } - /// Returns the pointer to the first byte of the linear memory for this instance. - pub fn base_ptr(&self) -> *const u8 { - self.memory.data_ptr() + /// Returns the virtual addresses that cover this instance's linear memory. + pub fn linear_memory_range(&self) -> Range { + let start = self.memory.data_ptr() as usize; + let end = start + self.memory.data_size(); + start..end } /// Removes physical backing from the allocated linear memory. This leads to returning the memory @@ -449,9 +451,66 @@ impl InstanceWrapper { }); } } + } else if #[cfg(target_os = "macos")] { + self.macos_decommit(); } } } + + /// Drop the instance's memory from the resident set. + /// + /// All access to this memory range will return `0` after doing so. + #[cfg(target_os = "macos")] + fn macos_decommit(&self) { + use mach::vm_purgable::{VM_PURGABLE_EMPTY, VM_PURGABLE_NONVOLATILE}; + + // # Safety + // + // We immediatly set the memory to non-volatile so that it will never be accessed + // in a volatile state. + unsafe { + // drop the instance's memory from the resident set + self.purge_control(VM_PURGABLE_EMPTY); + + // re-activate memory and prevent it from being automatically purged + self.purge_control(VM_PURGABLE_NONVOLATILE); + } + } + + /// Set the purgable state of the instance's memory mapping. + /// + /// # Safety + /// + /// - The caller must make sure to set the memmory to a non-volatile state before + /// any write will occur to it. + #[cfg(target_os = "macos")] + unsafe fn purge_control(&self, mut state: libc::c_int) { + use std::sync::Once; + use mach::{ + kern_return::KERN_SUCCESS, + traps::mach_task_self, + vm::mach_vm_purgable_control, + vm_purgable::{VM_PURGABLE_SET_STATE}, + }; + + // # Safety + // + // Unsafe because this a C-API. However, we make sure to pass in + // the correct address and assume that the caller uses proper values for + // `state`. + let result = mach_vm_purgable_control( + mach_task_self(), + self.memory.data_ptr() as _, + VM_PURGABLE_SET_STATE, + &mut state, + ); + if result != KERN_SUCCESS { + static LOGGED: Once = Once::new(); + LOGGED.call_once(|| { + log::warn!("mach_vm_purgeable_control({}) failed: {}", state, result); + }); + } + } } impl runtime_blob::InstanceGlobals for InstanceWrapper { diff --git a/client/executor/wasmtime/src/runtime.rs b/client/executor/wasmtime/src/runtime.rs index 5018b11264d71..f1f0ca8be7077 100644 --- a/client/executor/wasmtime/src/runtime.rs +++ b/client/executor/wasmtime/src/runtime.rs @@ -18,12 +18,15 @@ //! Defines the compiled Wasm runtime that uses Wasmtime internally. +#[cfg(target_os = "macos")] +mod mach_memory; + use crate::host::HostState; use crate::imports::{Imports, resolve_imports}; use crate::instance_wrapper::{InstanceWrapper, EntryPoint}; use crate::state_holder; -use std::{path::PathBuf, rc::Rc}; +use std::{path::PathBuf, rc::Rc, ops::Range}; use std::sync::Arc; use std::path::Path; use sc_executor_common::{ @@ -180,7 +183,7 @@ impl WasmInstance for WasmtimeInstance { } } - fn linear_memory_base_ptr(&self) -> Option<*const u8> { + fn linear_memory_range(&self) -> Option> { match &self.strategy { Strategy::RecreateInstance(_) => { // We do not keep the wasm instance around, therefore there is no linear memory @@ -189,7 +192,7 @@ impl WasmInstance for WasmtimeInstance { } Strategy::FastInstanceReuse { instance_wrapper, .. - } => Some(instance_wrapper.base_ptr()), + } => Some(instance_wrapper.linear_memory_range()), } } } @@ -235,6 +238,12 @@ directory = \"{cache_dir}\" fn common_config() -> wasmtime::Config { let mut config = wasmtime::Config::new(); config.cranelift_opt_level(wasmtime::OptLevel::SpeedAndSize); + + // On macOS we need to allocate the instance memory in a special way so that + // we can purge it when the instance is recycled during caching. + #[cfg(target_os = "macos")] + config.with_host_memory(Arc::new(mach_memory::Allocator::default())); + config } diff --git a/client/executor/wasmtime/src/runtime/mach_memory.rs b/client/executor/wasmtime/src/runtime/mach_memory.rs new file mode 100644 index 0000000000000..5a6a23aa0a3d6 --- /dev/null +++ b/client/executor/wasmtime/src/runtime/mach_memory.rs @@ -0,0 +1,522 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Defines a custom memory allocator for allocating host memory on the mach kernel. +//! This is needed in order to support purgable memory on macOS. + +use std::sync::atomic::{AtomicU32, Ordering}; +use bitflags::bitflags; +use libc::c_int; +use mach::{ + kern_return::KERN_SUCCESS, + traps::mach_task_self, + port::mach_port_t, + vm::{mach_vm_allocate, mach_vm_deallocate, mach_vm_protect}, + vm_prot::{vm_prot_t, VM_PROT_NONE, VM_PROT_DEFAULT}, + vm_page_size::vm_page_size, +}; +use wasmtime::{MemoryCreator, LinearMemory, MemoryType}; + +/// The size of wasm page. 2^WASM_PAGE_SHIFT = bytes per wasm page +const WASM_PAGE_SHIFT: u8 = 16; + +bitflags! { + /// Some of the flags that are allowed for `mach_vm_allocate`. + /// + /// https://github.com/apple/darwin-xnu/blob/main/osfmk/mach/vm_statistics.h + struct VmFlags: c_int { + const FIXED = 0x0000; + const ANYWHERE = 0x0001; + const PURGABLE = 0x0002; + const RANDOM_ADDR = 0x0008; + } +} + +/// Allocator that can allocate memory for wasmtime module instances. +pub struct Allocator { + /// The task that spawned the allocator. + task: mach_port_t, +} + +/// One memory area allocated by [`Allocator`]. +#[derive(Debug)] +pub struct Memory { + /// The task that allocated this memory. + task: mach_port_t, + /// The virtual address of the mapping. + address: u64, + /// The size of the mapping created in bytes. + /// + /// If this memory is grown beyond the virtual size we would need to allocate a new + /// a new mapping and copy over. However, this would only happen on 32bit systems + /// where not enough virtual space is available. Those systems are not supported + /// anyways for macOS. + /// + /// # Note + /// + /// This includes the guard bytes which are mapped but never accessible. + mapped_bytes: u64, + /// Size of the guard pages in bytes. + guard_bytes: u64, + /// The currently accesible number of wasm pages. + /// + /// Starting with wasmtime 0.28 we can remove the atomic as `LinearMemory::grow` has + /// an exclusive reference to this struct. + wasm_pages: AtomicU32, + /// The maximum number was wasm pages this memory is allowed to be growed to. + wasm_pages_max: Option, +} + +impl Allocator { + /// Allocate some memory and return the address to it in the `address` field. + fn allocate(&self, address: &mut u64, size: u64, flags: VmFlags) -> Result<(), String> { + // # Safety + // + // We do not allow passing of `VM_FLAGS_OVERWRITE` which would allow overwriting + // existing virtual regions. Other from that allocating memory is safe. + let result = unsafe { + // The `mach_vm` interface always returns page aligned addresses. + mach_vm_allocate(self.task, address, size, flags.bits()) + }; + if result == KERN_SUCCESS { + Ok(()) + } else { + Err(format!( + "mach_vm_allocate returned: {}. address: 0x{:016x} size: 0x{:016x} flags: {:?}", + result, address, size, flags, + )) + } + } +} + +impl Default for Allocator { + fn default() -> Self { + // # Safety + // + // There are no preconditions. It is unsafe only because it is a C-API. + let task = unsafe { mach_task_self() }; + Self { + task, + } + } +} + +unsafe impl MemoryCreator for Allocator { + fn new_memory( + &self, + ty: MemoryType, + reserved_bytes: Option, + guard_bytes: u64 + ) -> Result, String> { + let mapped_bytes = mapped_bytes(&ty, reserved_bytes, guard_bytes)?; + let anon_max_size = anon_max_size(); + + let mut address = 0; + self.allocate( + &mut address, + mapped_bytes.min(anon_max_size), + VmFlags::ANYWHERE | VmFlags::RANDOM_ADDR | VmFlags::PURGABLE, + )?; + + // Purgable mappings in macOS are at most `ANON_MAX_SIZE` large. We opt for a simple + // allocator where only the first mapping is purgable and all the rest is allocated + // unpurgable. When purging the memory only the first mapping is purged. This means + // as soon as any wasm instance uses more than `ANON_MAX_SIZE` memory anything above + // this threshold won't be purged. This is OK because we are not allowed to rely on + // purging semantics. It is merely to reduce the memory footprint. + // + // # Note + // + // `ANON_MAX_SIZE` is 4GB - 1 page. This means that at most one page won't be purged + // as memory sizes above 4GB don't make much sense. + if mapped_bytes > anon_max_size { + let mut address = address + anon_max_size; + self.allocate(&mut address, mapped_bytes - anon_max_size, VmFlags::FIXED)?; + } + + let memory = Box::new(Memory { + task: self.task, + address, + mapped_bytes, + guard_bytes, + wasm_pages: AtomicU32::new(ty.limits().min()), + wasm_pages_max: ty.limits().max(), + }); + + // We map all the reserved memory but only allow access to current the size of + // the memory from point of view of the wasm instance. + memory.protect()?; + + Ok(memory) + } +} + +impl Memory { + /// Change the memory permissions of the specified range. + /// + /// # Safety + /// + /// Caller must make sure that the supplied range and state won't interfere with + /// the correctness of the running program. + unsafe fn change_prot(&self, addr: u64, size: u64, prot: vm_prot_t) -> Result<(), String> { + let result = mach_vm_protect(self.task, addr, size, 0, prot); + if result == KERN_SUCCESS { + Ok(()) + } else { + Err(format!("mach_vm_protect returned: {}", result)) + } + } + + /// Free the specified memory. + /// + /// # Safety + /// + /// The caller must make sure that the memory is no longer in use. + unsafe fn free(&self, address: u64, size: u64) -> Result<(), String> { + let result = mach_vm_deallocate(self.task, address, size); + if result == KERN_SUCCESS { + Ok(()) + } else { + Err(format!( + "mach_vm_deallocate returned: {}. address: 0x{:016x} size: 0x{:016x}", + result, address, size + )) + } + } + + /// Returns number of currently accessible bytes. + fn accessible_bytes(&self) -> u64 { + (self.size() as u64) << WASM_PAGE_SHIFT + } + + /// Remove permissions to access currently unaccessible bytes. + fn protect(&self) -> Result<(), String> { + let accessible_bytes = self.accessible_bytes(); + + // # Safety + // + // We made sure when creating a memory that the calculated addresses are fully + // located within the instances memory. + unsafe { + self.change_prot( + self.address + accessible_bytes, + self.mapped_bytes - accessible_bytes, + VM_PROT_NONE, + ) + } + } + + /// Add permissions to access currently accessible bytes. + fn increase_accessible_bytes(&self) -> Option { + let accessible_bytes = self.accessible_bytes(); + + // We do not support 32bit applications on macOS. + assert!(accessible_bytes.checked_add(self.guard_bytes).unwrap() <= self.mapped_bytes, + "No memory relocation supported on macOS. This will only happen on 32bit systems."); + + // # Safety + // + // We made sure when creating a memory that the calculated addresses are fully + // located within the instances memory. + unsafe { + self.change_prot(self.address, accessible_bytes, VM_PROT_DEFAULT).ok()?; + } + + Some(accessible_bytes) + } +} + +unsafe impl LinearMemory for Memory { + fn size(&self) -> u32 { + self.wasm_pages.load(Ordering::Acquire) + } + + fn maximum(&self) -> Option { + self.wasm_pages_max + } + + fn grow(&self, delta: u32) -> Option { + self.wasm_pages.fetch_update(Ordering::SeqCst, Ordering:: SeqCst, |pages| { + let pages = pages.checked_add(delta)?; + if let Some(max) = self.wasm_pages_max { + if pages > max { + return None; + } + } + Some(pages) + }).ok()?; + + // All the memory is already mapped. We just need to allow access. + self.increase_accessible_bytes().map(|bytes| (bytes >> WASM_PAGE_SHIFT) as _) + } + + fn as_ptr(&self) -> *mut u8 { + self.address as _ + } +} + +impl Drop for Memory { + fn drop(&mut self) { + let anon_max_size = anon_max_size(); + + // # Safety + // + // The memory got dropped which means no reference to the memory exist. Therefore + // it is no longer in use and can be freed safely. + let result = unsafe { + self.free(self.address, self.mapped_bytes.min(anon_max_size)).and_then(|_| { + if self.mapped_bytes > anon_max_size { + self.free(self.address + anon_max_size, self.mapped_bytes - anon_max_size) + } else { + Ok(()) + } + }) + }; + + if let Err(err) = result { + log::error!("deallocating instance memory failed with: {}.\n{:#?}", err, self); + } + } +} + +/// Max size of a purgable mapping in bytes. +/// +/// https://github.com/apple/darwin-xnu/blob/main/osfmk/mach/vm_param.h +fn anon_max_size() -> u64 { + (1u64 << 32) - page_size() +} + +fn page_size() -> u64 { + // # Safety + // + // There are no preconditions. It is unsafe only because it is a C-API. + (unsafe { vm_page_size }) as u64 +} + +fn mapped_bytes(ty: &MemoryType, reserved: Option, guard: u64) -> Result { + let accessible_bytes = (u64::from(ty.limits().min())) << WASM_PAGE_SHIFT; + let mapped_bytes = if let Some(reserved) = reserved { + reserved.max(accessible_bytes) + } else { + accessible_bytes + } + .checked_add(guard) + .ok_or_else(|| "Guard size overflowed u64".to_string())?; + Ok(mapped_bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ops::Range, fmt::self}; + use sc_executor_common::test_utils::{get_regions, Region}; + use wasmtime::Limits; + + /// The default reserved value used by wasmtime on 64bit systems in order to + /// reserve the whole wasm address space and make any re-allocation unnecessary. + const DEFAULT_RESERVED: u64 = 4 * 1024 * 1024 * 1024; + + /// The default guard size used by wasmtime on 64bit systems in order to completely + /// eliminate bound checks. + const DEFAULT_GUARD: u64 = 2 * 1024 * 1024 * 1024; + + struct MemInfo { + memory: Box, + range: Range, + } + + impl MemInfo { + fn new(ty: MemoryType, reserved: Option, guard: u64) -> Self { + let allocator = Allocator::default(); + let memory = allocator.new_memory(ty.clone(), reserved, guard).unwrap(); + let start = memory.as_ptr() as u64; + let mapped_bytes = mapped_bytes(&ty, reserved, guard).unwrap(); + let range = start..(start + mapped_bytes); + let result = Self { + memory, + range, + }; + result.check_memory_map(); + result + } + + fn check_memory_map(&self) { + let max_size = anon_max_size(); + let regions = self.regions(); + let mapped_bytes = self.range.end - self.range.start; + let accessible_bytes = (self.memory.size() as u64) << WASM_PAGE_SHIFT; + let accessible_regions = if accessible_bytes > max_size { + 2 + } else { + 1 + }; + + // memory map should start with the first requested address + assert_eq!(regions[0].range.start, self.range.start); + + // check that accessible regions have the correct protection flags (3 = read/write) + for i in 0..accessible_regions { + assert_eq!(regions[i].info.protection, 3); + } + + // accessible regions should cover the accessible bytes + assert_eq!(regions[0].len(), accessible_bytes.min(max_size)); + if accessible_bytes > max_size { + assert_eq!(regions[1].len(), accessible_bytes - max_size); + } + + // all other regions shouldn't be accessible (0 = no access) + assert!(regions.iter().skip(accessible_regions).all(|r| r.info.protection == 0)); + + // the other regions should cover the rest of the mapped memory + assert_eq!(regions.last().unwrap().range.end, self.range.end); + assert_eq!( + regions.iter().skip(accessible_regions).map(|r| r.len()).sum::(), + mapped_bytes - accessible_bytes, + ); + } + + fn regions(&self) -> Vec { + get_regions(self.range.clone()).unwrap() + } + } + + impl fmt::Display for MemInfo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{:016x?} - {} pages accessible", self.range, self.memory.size())?; + writeln!(f, "--------------------------------------------------------------")?; + for region in self.regions() { + writeln!(f, "{}", region)?; + } + writeln!(f, "--------------------------------------------------------------") + } + } + + #[test] + fn alloc_works() { + let ty = MemoryType::new(Limits::new(1, Some(5))); + let info = MemInfo::new(ty, Some(DEFAULT_RESERVED), DEFAULT_GUARD); + assert_eq!(info.memory.size(), 1); + assert_eq!(info.memory.maximum(), Some(5)); + } + + #[test] + fn grow_works() { + let ty = MemoryType::new(Limits::at_least(1)); + let info = MemInfo::new(ty, Some(DEFAULT_RESERVED), DEFAULT_GUARD); + assert_eq!(info.memory.size(), 1); + assert_eq!(info.memory.maximum(), None); + info.check_memory_map(); + info.memory.grow(12).expect("Grow should work."); + assert_eq!(info.memory.size(), 13); + } + + #[test] + fn max_alloc_works() { + let ty = MemoryType::new(Limits::at_least(65536)); + let info = MemInfo::new(ty, Some(DEFAULT_RESERVED), DEFAULT_GUARD); + assert_eq!(info.memory.size(), 65536); + assert_eq!(info.memory.maximum(), None); + } + + #[test] + fn grow_beyond_max_fails() { + let ty = MemoryType::new(Limits::new(2, Some(5))); + let info = MemInfo::new(ty, Some(DEFAULT_RESERVED), DEFAULT_GUARD); + assert_eq!(info.memory.size(), 2); + assert_eq!(info.memory.maximum(), Some(5)); + + info.memory.grow(3).expect("2 + 3 pages are within the allowed range"); + + // Grows to 6 which is larger than the allowed maximum + assert!(info.memory.grow(1).is_none()); + + // Size shouldn't have changed + assert_eq!(info.memory.size(), 5); + assert_eq!(info.memory.maximum(), Some(5)); + info.check_memory_map(); + } + + #[test] + fn no_guard_page_works() { + let ty = MemoryType::new(Limits::at_least(65536)); + let info = MemInfo::new(ty, Some(DEFAULT_RESERVED), 0); + assert_eq!(info.memory.size(), 65536); + assert_eq!(info.memory.maximum(), None); + } + + #[test] + fn initial_size_larger_than_reserve_works() { + let ty = MemoryType::new(Limits::at_least(1)); + let info = MemInfo::new(ty, Some(4096), 0); + assert_eq!(info.memory.size(), 1); + assert_eq!(info.memory.maximum(), None); + } + + #[test] + fn initial_size_larger_than_reserve_with_guard_works() { + let ty = MemoryType::new(Limits::new(5, Some(10))); + let info = MemInfo::new(ty, Some(4096), DEFAULT_GUARD); + assert_eq!(info.memory.size(), 5); + assert_eq!(info.memory.maximum(), Some(10)); + } + + #[test] + fn initial_size_no_reserved_works() { + let ty = MemoryType::new(Limits::at_least(1)); + let info = MemInfo::new(ty, None, 0); + assert_eq!(info.memory.size(), 1); + assert_eq!(info.memory.maximum(), None); + } + + #[test] + fn drop_works() { + let ty = MemoryType::new(Limits::at_least(1)); + let info = MemInfo::new(ty, Some(DEFAULT_RESERVED), DEFAULT_GUARD); + assert_eq!(info.memory.size(), 1); + assert_eq!(info.memory.maximum(), None); + + // we need this information later after the drop + let range = info.range.clone(); + drop(info); + + // drop should have removed all memory mappings + assert_eq!(get_regions(range).map(|r| r.len()).unwrap_or(0), 0); + } + + #[test] + #[should_panic(expected = "No memory relocation supported on macOS. This will only happen on 32bit systems.")] + fn grow_more_than_reserved_bytes_panics_0() { + let info = MemInfo::new(MemoryType::new(Limits::at_least(1)), Some(3 * 64 * 1024), 0); + info.memory.grow(3); + } + + #[test] + #[should_panic(expected = "No memory relocation supported on macOS. This will only happen on 32bit systems.")] + fn grow_more_than_reserved_bytes_panics_1() { + let info = MemInfo::new(MemoryType::new(Limits::at_least(3)), None, 0); + info.memory.grow(1); + } + + #[test] + #[should_panic(expected = "No memory relocation supported on macOS. This will only happen on 32bit systems.")] + fn grow_more_than_reserved_bytes_panics_2() { + let info = MemInfo::new(MemoryType::new(Limits::at_least(3)), None, DEFAULT_GUARD); + info.memory.grow(1); + } +}