diff --git a/crates/wasmi/src/engine/executor.rs b/crates/wasmi/src/engine/executor.rs index 4107aaa1fd..c608ceb092 100644 --- a/crates/wasmi/src/engine/executor.rs +++ b/crates/wasmi/src/engine/executor.rs @@ -1,32 +1,33 @@ -use super::{ - super::{Memory, Table}, - bytecode::{ - BranchParams, - DataSegmentIdx, - ElementSegmentIdx, - FuncIdx, - GlobalIdx, - Instruction, - LocalDepth, - Offset, - SignatureIdx, - TableIdx, - }, - cache::InstanceCache, - code_map::InstructionPtr, - stack::ValueStackRef, - CallOutcome, - DropKeep, - FuncFrame, - ValueStack, -}; use crate::{ core::TrapCode, - engine::config::FuelCosts, + engine::{ + bytecode::{ + BranchParams, + DataSegmentIdx, + ElementSegmentIdx, + FuncIdx, + GlobalIdx, + Instruction, + LocalDepth, + Offset, + SignatureIdx, + TableIdx, + }, + cache::InstanceCache, + code_map::InstructionPtr, + config::FuelCosts, + stack::ValueStackPtr, + CallOutcome, + DropKeep, + FuncFrame, + ValueStack, + }, table::TableEntity, Func, FuncRef, + Memory, StoreInner, + Table, }; use core::cmp::{self}; use wasmi_core::{Pages, UntypedValue}; @@ -88,7 +89,14 @@ struct Executor<'ctx, 'engine, 'func> { /// The pointer to the currently executed instruction. ip: InstructionPtr, /// Stores the value stack of live values on the Wasm stack. - value_stack: ValueStackRef<'engine>, + sp: ValueStackPtr, + /// The value stack. + /// + /// # Note + /// + /// This reference is mainly used to synchronize back state + /// after manipulations to the value stack via `sp`. + value_stack: &'engine mut ValueStack, /// A mutable [`StoreInner`] context. /// /// [`StoreInner`]: [`crate::StoreInner`] @@ -110,9 +118,10 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { ) -> Self { cache.update_instance(frame.instance()); let ip = frame.ip(); - let value_stack = ValueStackRef::new(value_stack); + let sp = value_stack.stack_ptr(); Self { ip, + sp, value_stack, ctx, cache, @@ -370,7 +379,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { offset: Offset, load_extend: WasmLoadOp, ) -> Result<(), TrapCode> { - self.value_stack.try_eval_top(|address| { + self.sp.try_eval_top(|address| { let memory = self.cache.default_memory_bytes(self.ctx).data(); let value = load_extend(memory, address, offset.into_inner())?; Ok(value) @@ -393,43 +402,48 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { offset: Offset, store_wrap: WasmStoreOp, ) -> Result<(), TrapCode> { - let (address, value) = self.value_stack.pop2(); + let (address, value) = self.sp.pop2(); let memory = self.cache.default_memory_bytes(self.ctx).data_mut(); store_wrap(memory, address, offset.into_inner(), value)?; self.try_next_instr() } /// Executes an infallible unary `wasmi` instruction. + #[inline] fn execute_unary(&mut self, f: fn(UntypedValue) -> UntypedValue) { - self.value_stack.eval_top(f); + self.sp.eval_top(f); self.next_instr() } /// Executes a fallible unary `wasmi` instruction. + #[inline] fn try_execute_unary( &mut self, f: fn(UntypedValue) -> Result, ) -> Result<(), TrapCode> { - self.value_stack.try_eval_top(f)?; + self.sp.try_eval_top(f)?; self.try_next_instr() } /// Executes an infallible binary `wasmi` instruction. + #[inline] fn execute_binary(&mut self, f: fn(UntypedValue, UntypedValue) -> UntypedValue) { - self.value_stack.eval_top2(f); + self.sp.eval_top2(f); self.next_instr() } /// Executes a fallible binary `wasmi` instruction. + #[inline] fn try_execute_binary( &mut self, f: fn(UntypedValue, UntypedValue) -> Result, ) -> Result<(), TrapCode> { - self.value_stack.try_eval_top2(f)?; + self.sp.try_eval_top2(f)?; self.try_next_instr() } /// Shifts the instruction pointer to the next instruction. + #[inline] fn next_instr(&mut self) { self.ip_add(1) } @@ -439,17 +453,21 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { /// # Note /// /// This is a convenience function for fallible instructions. + #[inline] fn try_next_instr(&mut self) -> Result<(), TrapCode> { self.next_instr(); Ok(()) } /// Offsets the instruction pointer using the given [`BranchParams`]. + #[inline] fn branch_to(&mut self, params: BranchParams) { - self.value_stack.drop_keep(params.drop_keep()); + self.sp.drop_keep(params.drop_keep()); self.ip_add(params.offset().into_i32() as isize) } + /// Adjusts the [`InstructionPtr`] by `delta` in terms of [`Instruction`]. + #[inline] fn ip_add(&mut self, delta: isize) { // Safety: This is safe since we carefully constructed the `wasmi` // bytecode in conjunction with Wasm validation so that the @@ -469,8 +487,9 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { /// For performance reasons we detach the stack pointer form the [`ValueStack`]. /// Therefore it is necessary to synchronize the [`ValueStack`] upon finishing /// execution of a sequence of non control flow instructions. + #[inline] fn sync_stack_ptr(&mut self) { - self.value_stack.sync(); + self.value_stack.sync_stack_ptr(self.sp); } /// Calls the given [`Func`]. @@ -490,7 +509,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { /// This also modifies the stack as the caller would expect it /// and synchronizes the execution state with the outer structures. fn ret(&mut self, drop_keep: DropKeep) { - self.value_stack.drop_keep(drop_keep); + self.sp.drop_keep(drop_keep); self.sync_stack_ptr(); } @@ -571,7 +590,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { } fn visit_br_if_eqz(&mut self, params: BranchParams) { - let condition = self.value_stack.pop_as(); + let condition = self.sp.pop_as(); if condition { self.next_instr() } else { @@ -580,7 +599,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { } fn visit_br_if_nez(&mut self, params: BranchParams) { - let condition = self.value_stack.pop_as(); + let condition = self.sp.pop_as(); if condition { self.branch_to(params) } else { @@ -589,7 +608,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { } fn visit_return_if_nez(&mut self, drop_keep: DropKeep) -> MaybeReturn { - let condition = self.value_stack.pop_as(); + let condition = self.sp.pop_as(); if condition { self.ret(drop_keep); MaybeReturn::Return @@ -600,7 +619,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { } fn visit_br_table(&mut self, len_targets: usize) { - let index: u32 = self.value_stack.pop_as(); + let index: u32 = self.sp.pop_as(); // The index of the default target which is the last target of the slice. let max_index = len_targets - 1; // A normalized index will always yield a target without panicking. @@ -617,31 +636,31 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { } fn visit_local_get(&mut self, local_depth: LocalDepth) { - let value = self.value_stack.peek(local_depth.into_inner()); - self.value_stack.push(value); + let value = self.sp.nth_back(local_depth.into_inner()); + self.sp.push(value); self.next_instr() } fn visit_local_set(&mut self, local_depth: LocalDepth) { - let new_value = self.value_stack.pop(); - *self.value_stack.peek_mut(local_depth.into_inner()) = new_value; + let new_value = self.sp.pop(); + self.sp.set_nth_back(local_depth.into_inner(), new_value); self.next_instr() } fn visit_local_tee(&mut self, local_depth: LocalDepth) { - let new_value = self.value_stack.last(); - *self.value_stack.peek_mut(local_depth.into_inner()) = new_value; + let new_value = self.sp.last(); + self.sp.set_nth_back(local_depth.into_inner(), new_value); self.next_instr() } fn visit_global_get(&mut self, global_index: GlobalIdx) { let global_value = *self.global(global_index); - self.value_stack.push(global_value); + self.sp.push(global_value); self.next_instr() } fn visit_global_set(&mut self, global_index: GlobalIdx) { - let new_value = self.value_stack.pop(); + let new_value = self.sp.pop(); *self.global(global_index) = new_value; self.next_instr() } @@ -656,7 +675,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { table: TableIdx, func_type: SignatureIdx, ) -> Result { - let func_index: u32 = self.value_stack.pop_as(); + let func_index: u32 = self.sp.pop_as(); let table = self.cache.get_table(self.ctx, table); let funcref = self .ctx @@ -680,17 +699,17 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { } fn visit_const(&mut self, bytes: UntypedValue) { - self.value_stack.push(bytes); + self.sp.push(bytes); self.next_instr() } fn visit_drop(&mut self) { - let _ = self.value_stack.pop(); + self.sp.drop(); self.next_instr() } fn visit_select(&mut self) { - self.value_stack.eval_top3(|e1, e2, e3| { + self.sp.eval_top3(|e1, e2, e3| { let condition = >::from(e3); if condition { e1 @@ -704,18 +723,18 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { fn visit_memory_size(&mut self) { let memory = self.default_memory(); let result: u32 = self.ctx.resolve_memory(&memory).current_pages().into(); - self.value_stack.push(result); + self.sp.push_as(result); self.next_instr() } fn visit_memory_grow(&mut self) -> Result<(), TrapCode> { let memory = self.default_memory(); - let delta: u32 = self.value_stack.pop_as(); + let delta: u32 = self.sp.pop_as(); let delta = match Pages::new(delta) { Some(pages) => pages, None => { // Cannot grow memory so we push the expected error value. - self.value_stack.push(INVALID_GROWTH_ERRCODE); + self.sp.push_as(INVALID_GROWTH_ERRCODE); return self.try_next_instr(); } }; @@ -743,13 +762,13 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { Err(EntityGrowError::InvalidGrow) => INVALID_GROWTH_ERRCODE, Err(EntityGrowError::TrapCode(trap_code)) => return Err(trap_code), }; - self.value_stack.push(result); + self.sp.push_as(result); self.try_next_instr() } fn visit_memory_fill(&mut self) -> Result<(), TrapCode> { // The `n`, `val` and `d` variable bindings are extracted from the Wasm specification. - let (d, val, n) = self.value_stack.pop3(); + let (d, val, n) = self.sp.pop3(); let n = i32::from(n) as usize; let offset = i32::from(d) as usize; let byte = u8::from(val); @@ -771,7 +790,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { fn visit_memory_copy(&mut self) -> Result<(), TrapCode> { // The `n`, `s` and `d` variable bindings are extracted from the Wasm specification. - let (d, s, n) = self.value_stack.pop3(); + let (d, s, n) = self.sp.pop3(); let n = i32::from(n) as usize; let src_offset = i32::from(s) as usize; let dst_offset = i32::from(d) as usize; @@ -795,7 +814,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { fn visit_memory_init(&mut self, segment: DataSegmentIdx) -> Result<(), TrapCode> { // The `n`, `s` and `d` variable bindings are extracted from the Wasm specification. - let (d, s, n) = self.value_stack.pop3(); + let (d, s, n) = self.sp.pop3(); let n = i32::from(n) as usize; let src_offset = i32::from(s) as usize; let dst_offset = i32::from(d) as usize; @@ -831,12 +850,12 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { fn visit_table_size(&mut self, table_index: TableIdx) { let table = self.cache.get_table(self.ctx, table_index); let size = self.ctx.resolve_table(&table).size(); - self.value_stack.push(size); + self.sp.push_as(size); self.next_instr() } fn visit_table_grow(&mut self, table_index: TableIdx) -> Result<(), TrapCode> { - let (init, delta) = self.value_stack.pop2(); + let (init, delta) = self.sp.pop2(); let delta: u32 = delta.into(); let result = self.consume_fuel_on_success( |costs| u64::from(delta) * costs.table_per_element, @@ -853,13 +872,13 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { Err(EntityGrowError::InvalidGrow) => INVALID_GROWTH_ERRCODE, Err(EntityGrowError::TrapCode(trap_code)) => return Err(trap_code), }; - self.value_stack.push(result); + self.sp.push_as(result); self.try_next_instr() } fn visit_table_fill(&mut self, table_index: TableIdx) -> Result<(), TrapCode> { // The `n`, `s` and `d` variable bindings are extracted from the Wasm specification. - let (i, val, n) = self.value_stack.pop3(); + let (i, val, n) = self.sp.pop3(); let dst: u32 = i.into(); let len: u32 = n.into(); self.consume_fuel_on_success( @@ -876,7 +895,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { } fn visit_table_get(&mut self, table_index: TableIdx) -> Result<(), TrapCode> { - self.value_stack.try_eval_top(|index| { + self.sp.try_eval_top(|index| { let index: u32 = index.into(); let table = self.cache.get_table(self.ctx, table_index); self.ctx @@ -888,7 +907,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { } fn visit_table_set(&mut self, table_index: TableIdx) -> Result<(), TrapCode> { - let (index, value) = self.value_stack.pop2(); + let (index, value) = self.sp.pop2(); let index: u32 = index.into(); let table = self.cache.get_table(self.ctx, table_index); self.ctx @@ -900,7 +919,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { fn visit_table_copy(&mut self, dst: TableIdx, src: TableIdx) -> Result<(), TrapCode> { // The `n`, `s` and `d` variable bindings are extracted from the Wasm specification. - let (d, s, n) = self.value_stack.pop3(); + let (d, s, n) = self.sp.pop3(); let len = u32::from(n); let src_index = u32::from(s); let dst_index = u32::from(d); @@ -931,7 +950,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { elem: ElementSegmentIdx, ) -> Result<(), TrapCode> { // The `n`, `s` and `d` variable bindings are extracted from the Wasm specification. - let (d, s, n) = self.value_stack.pop3(); + let (d, s, n) = self.sp.pop3(); let len = u32::from(n); let src_index = u32::from(s); let dst_index = u32::from(d); @@ -961,7 +980,7 @@ impl<'ctx, 'engine, 'func> Executor<'ctx, 'engine, 'func> { fn visit_ref_func(&mut self, func_index: FuncIdx) { let func = self.cache.get_func(self.ctx, func_index.into_inner()); let funcref = FuncRef::new(func); - self.value_stack.push(funcref); + self.sp.push_as(funcref); self.next_instr(); } diff --git a/crates/wasmi/src/engine/stack/mod.rs b/crates/wasmi/src/engine/stack/mod.rs index 2dac4bf70a..457f176453 100644 --- a/crates/wasmi/src/engine/stack/mod.rs +++ b/crates/wasmi/src/engine/stack/mod.rs @@ -3,7 +3,7 @@ mod values; pub use self::{ frames::{CallStack, FuncFrame}, - values::{ValueStack, ValueStackRef}, + values::{ValueStack, ValueStackPtr}, }; use super::{ code_map::{CodeMap, InstructionPtr}, diff --git a/crates/wasmi/src/engine/stack/values/mod.rs b/crates/wasmi/src/engine/stack/values/mod.rs index 1aa6bd07b6..e2b49dd095 100644 --- a/crates/wasmi/src/engine/stack/values/mod.rs +++ b/crates/wasmi/src/engine/stack/values/mod.rs @@ -1,11 +1,11 @@ //! Data structures to represent the Wasm value stack during execution. -mod vref; +mod sp; #[cfg(test)] mod tests; -pub use self::vref::ValueStackRef; +pub use self::sp::ValueStackPtr; use super::{err_stack_overflow, DEFAULT_MAX_VALUE_STACK_HEIGHT, DEFAULT_MIN_VALUE_STACK_HEIGHT}; use crate::core::TrapCode; use alloc::vec::Vec; @@ -83,6 +83,16 @@ impl FromIterator for ValueStack { } } +#[cfg(test)] +impl<'a> IntoIterator for &'a ValueStack { + type Item = &'a UntypedValue; + type IntoIter = core::slice::Iter<'a, UntypedValue>; + + fn into_iter(self) -> Self::IntoIter { + self.entries[0..self.stack_ptr].iter() + } +} + impl ValueStack { /// Creates an empty [`ValueStack`] that does not allocate heap memor. /// @@ -98,6 +108,34 @@ impl ValueStack { } } + #[cfg(test)] + pub fn iter(&self) -> core::slice::Iter { + self.into_iter() + } + + /// Returns the current [`ValueStackPtr`] of `self`. + /// + /// The returned [`ValueStackPtr`] points to the top most value on the [`ValueStack`]. + #[inline] + pub fn stack_ptr(&mut self) -> ValueStackPtr { + self.base_ptr().into_add(self.stack_ptr) + } + + /// Returns the base [`ValueStackPtr`] of `self`. + /// + /// The returned [`ValueStackPtr`] points to the first value on the [`ValueStack`]. + #[inline] + fn base_ptr(&mut self) -> ValueStackPtr { + ValueStackPtr::from(self.entries.as_mut_ptr()) + } + + /// Synchronizes [`ValueStack`] with the new [`ValueStackPtr`]. + #[inline] + pub fn sync_stack_ptr(&mut self, new_sp: ValueStackPtr) { + let offset = new_sp.offset_from(self.base_ptr()); + self.stack_ptr = offset as usize; + } + /// Returns `true` if the [`ValueStack`] is empty. pub fn is_empty(&self) -> bool { self.entries.capacity() == 0 @@ -201,8 +239,8 @@ impl ValueStack { /// /// # Note /// - /// This allows efficient implementation of [`ValueStack::push`] and - /// [`ValueStackRef::pop`] operations. + /// This allows to efficiently operate on the [`ValueStack`] through + /// [`ValueStackPtr`] which requires external resource management. /// /// Before executing a function the interpreter calls this function /// to guarantee that enough space on the [`ValueStack`] exists for diff --git a/crates/wasmi/src/engine/stack/values/sp.rs b/crates/wasmi/src/engine/stack/values/sp.rs new file mode 100644 index 0000000000..831ed73b48 --- /dev/null +++ b/crates/wasmi/src/engine/stack/values/sp.rs @@ -0,0 +1,330 @@ +use crate::{ + core::{TrapCode, UntypedValue}, + engine::DropKeep, +}; + +/// A pointer on the [`ValueStack`]. +/// +/// Allows for efficient mutable access to the values of the [`ValueStack`]. +/// +/// [`ValueStack`]: super::ValueStack +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub struct ValueStackPtr { + ptr: *mut UntypedValue, +} + +impl From<*mut UntypedValue> for ValueStackPtr { + #[inline] + fn from(ptr: *mut UntypedValue) -> Self { + Self { ptr } + } +} + +impl ValueStackPtr { + /// Calculates the distance between two [`ValueStackPtr] in units of [`UntypedValue`]. + #[inline] + pub fn offset_from(self, other: Self) -> isize { + unsafe { self.ptr.offset_from(other.ptr) } + } + + /// Returns the [`UntypedValue`] at the current stack pointer. + #[must_use] + #[inline] + fn get(self) -> UntypedValue { + unsafe { *self.ptr } + } + + /// Writes `value` to the cell pointed at by [`ValueStackPtr`]. + #[inline] + fn set(self, value: UntypedValue) { + *unsafe { &mut *self.ptr } = value; + } + + /// Returns a [`ValueStackPtr`] with a pointer value increased by `delta`. + /// + /// # Note + /// + /// The amount of `delta` is in number of bytes per [`UntypedValue`]. + #[must_use] + #[inline] + pub fn into_add(mut self, delta: usize) -> Self { + self.inc_by(delta); + self + } + + /// Returns a [`ValueStackPtr`] with a pointer value decreased by `delta`. + /// + /// # Note + /// + /// The amount of `delta` is in number of bytes per [`UntypedValue`]. + #[must_use] + #[inline] + pub fn into_sub(mut self, delta: usize) -> Self { + self.dec_by(delta); + self + } + + /// Returns the last [`UntypedValue`] on the [`ValueStack`]. + /// + /// # Note + /// + /// This has the same effect as [`ValueStackPtr::nth_back`]`(1)`. + /// + /// [`ValueStack`]: super::ValueStack + #[inline] + #[must_use] + pub fn last(self) -> UntypedValue { + self.nth_back(1) + } + + /// Peeks the entry at the given depth from the last entry. + /// + /// # Note + /// + /// Given a `depth` of 1 has the same effect as [`ValueStackPtr::last`]. + /// + /// A `depth` of 0 is invalid and undefined. + #[inline] + #[must_use] + pub fn nth_back(self, depth: usize) -> UntypedValue { + self.into_sub(depth).get() + } + + /// Writes `value` to the n-th [`UntypedValue`] from the back. + /// + /// # Note + /// + /// Given a `depth` of 1 has the same effect as mutating [`ValueStackPtr::last`]. + /// + /// A `depth` of 0 is invalid and undefined. + #[inline] + pub fn set_nth_back(self, depth: usize, value: UntypedValue) { + self.into_sub(depth).set(value) + } + + /// Bumps the [`ValueStackPtr`] of `self` by one. + #[inline] + fn inc_by(&mut self, delta: usize) { + self.ptr = unsafe { self.ptr.add(delta) }; + } + + /// Decreases the [`ValueStackPtr`] of `self` by one. + #[inline] + fn dec_by(&mut self, delta: usize) { + self.ptr = unsafe { self.ptr.sub(delta) }; + } + + /// Pushes the `T` to the end of the [`ValueStack`]. + /// + /// # Note + /// + /// - This operation heavily relies on the prior validation of + /// the executed WebAssembly bytecode for correctness. + /// - Especially the stack-depth analysis during compilation with + /// a manual stack extension before function call prevents this + /// procedure from panicking. + /// + /// [`ValueStack`]: super::ValueStack + #[inline] + pub fn push_as(&mut self, value: T) + where + T: Into, + { + self.push(value.into()) + } + + /// Pushes the [`UntypedValue`] to the end of the [`ValueStack`]. + /// + /// # Note + /// + /// - This operation heavily relies on the prior validation of + /// the executed WebAssembly bytecode for correctness. + /// - Especially the stack-depth analysis during compilation with + /// a manual stack extension before function call prevents this + /// procedure from panicking. + /// + /// [`ValueStack`]: super::ValueStack + #[inline] + pub fn push(&mut self, value: UntypedValue) { + self.set(value); + self.inc_by(1); + } + + /// Drops the last [`UntypedValue`] from the [`ValueStack`]. + /// + /// # Note + /// + /// This operation heavily relies on the prior validation of + /// the executed WebAssembly bytecode for correctness. + /// + /// [`ValueStack`]: super::ValueStack + #[inline] + pub fn drop(&mut self) { + self.dec_by(1); + } + + /// Pops the last [`UntypedValue`] from the [`ValueStack`] as `T`. + /// + /// [`ValueStack`]: super::ValueStack + #[inline] + pub fn pop_as(&mut self) -> T + where + T: From, + { + T::from(self.pop()) + } + + /// Pops the last [`UntypedValue`] from the [`ValueStack`]. + /// + /// # Note + /// + /// This operation heavily relies on the prior validation of + /// the executed WebAssembly bytecode for correctness. + /// + /// [`ValueStack`]: super::ValueStack + #[inline] + pub fn pop(&mut self) -> UntypedValue { + self.dec_by(1); + self.get() + } + + /// Pops the last pair of [`UntypedValue`] from the [`ValueStack`]. + /// + /// # Note + /// + /// This operation heavily relies on the prior validation of + /// the executed WebAssembly bytecode for correctness. + /// + /// [`ValueStack`]: super::ValueStack + #[inline] + pub fn pop2(&mut self) -> (UntypedValue, UntypedValue) { + let rhs = self.pop(); + let lhs = self.pop(); + (lhs, rhs) + } + + /// Pops the last triple of [`UntypedValue`] from the [`ValueStack`]. + /// + /// # Note + /// + /// This operation heavily relies on the prior validation of + /// the executed WebAssembly bytecode for correctness. + /// + /// [`ValueStack`]: super::ValueStack + #[inline] + pub fn pop3(&mut self) -> (UntypedValue, UntypedValue, UntypedValue) { + let (snd, trd) = self.pop2(); + let fst = self.pop(); + (fst, snd, trd) + } + + /// Evaluates the given closure `f` for the top most stack value. + #[inline] + pub fn eval_top(&mut self, f: F) + where + F: FnOnce(UntypedValue) -> UntypedValue, + { + let last = self.into_sub(1); + last.set(f(last.get())) + } + + /// Evaluates the given closure `f` for the 2 top most stack values. + #[inline] + pub fn eval_top2(&mut self, f: F) + where + F: FnOnce(UntypedValue, UntypedValue) -> UntypedValue, + { + let rhs = self.pop(); + let last = self.into_sub(1); + let lhs = last.get(); + last.set(f(lhs, rhs)); + } + + /// Evaluates the given closure `f` for the 3 top most stack values. + #[inline] + pub fn eval_top3(&mut self, f: F) + where + F: FnOnce(UntypedValue, UntypedValue, UntypedValue) -> UntypedValue, + { + let (e2, e3) = self.pop2(); + let last = self.into_sub(1); + let e1 = last.get(); + last.set(f(e1, e2, e3)); + } + + /// Evaluates the given fallible closure `f` for the top most stack value. + /// + /// # Errors + /// + /// If the closure execution fails. + #[inline] + pub fn try_eval_top(&mut self, f: F) -> Result<(), TrapCode> + where + F: FnOnce(UntypedValue) -> Result, + { + let last = self.into_sub(1); + last.set(f(last.get())?); + Ok(()) + } + + /// Evaluates the given fallible closure `f` for the 2 top most stack values. + /// + /// # Errors + /// + /// If the closure execution fails. + #[inline] + pub fn try_eval_top2(&mut self, f: F) -> Result<(), TrapCode> + where + F: FnOnce(UntypedValue, UntypedValue) -> Result, + { + let rhs = self.pop(); + let last = self.into_sub(1); + let lhs = last.get(); + last.set(f(lhs, rhs)?); + Ok(()) + } + + /// Drops some amount of entries and keeps some amount of them at the new top. + /// + /// # Note + /// + /// For an amount of entries to keep `k` and an amount of entries to drop `d` + /// this has the following effect on stack `s` and stack pointer `sp`. + /// + /// 1) Copy `k` elements from indices starting at `sp - k` to `sp - k - d`. + /// 2) Adjust stack pointer: `sp -= d` + /// + /// After this operation the value stack will have `d` fewer entries and the + /// top `k` entries are the top `k` entries before this operation. + /// + /// Note that `k + d` cannot be greater than the stack length. + pub fn drop_keep(&mut self, drop_keep: DropKeep) { + fn drop_keep_impl(this: ValueStackPtr, drop_keep: DropKeep) { + let keep = drop_keep.keep(); + if keep == 0 { + // Case: no values need to be kept. + return; + } + let src = this.into_sub(keep); + let dst = this.into_sub(keep + drop_keep.drop()); + if keep == 1 { + // Case: only one value needs to be kept. + dst.set(src.get()); + return; + } + // Case: many values need to be kept and moved on the stack. + for i in 0..keep { + dst.into_add(i).set(src.into_add(i).get()); + } + } + + let drop = drop_keep.drop(); + if drop == 0 { + // Nothing to do in this case. + return; + } + drop_keep_impl(*self, drop_keep); + self.dec_by(drop); + } +} diff --git a/crates/wasmi/src/engine/stack/values/tests.rs b/crates/wasmi/src/engine/stack/values/tests.rs index f2163e515e..f8411a05dd 100644 --- a/crates/wasmi/src/engine/stack/values/tests.rs +++ b/crates/wasmi/src/engine/stack/values/tests.rs @@ -5,21 +5,6 @@ fn drop_keep(drop: usize, keep: usize) -> DropKeep { DropKeep::new(drop, keep).unwrap() } -impl<'a> IntoIterator for &'a ValueStackRef<'a> { - type Item = &'a UntypedValue; - type IntoIter = core::slice::Iter<'a, UntypedValue>; - - fn into_iter(self) -> Self::IntoIter { - self.values[0..self.stack_ptr].iter() - } -} - -impl<'a> ValueStackRef<'a> { - pub fn iter(&'a self) -> core::slice::Iter<'a, UntypedValue> { - self.into_iter() - } -} - #[test] fn drop_keep_works() { fn assert_drop_keep(stack: &ValueStack, drop_keep: DropKeep, expected: E) @@ -28,10 +13,12 @@ fn drop_keep_works() { E::Item: Into, { let mut s = stack.clone(); - let mut s = ValueStackRef::new(&mut s); - s.drop_keep(drop_keep); - let expected = expected.into_iter().map(Into::into); - assert!(s.iter().copied().eq(expected)); + let mut sp = s.stack_ptr(); + sp.drop_keep(drop_keep); + s.sync_stack_ptr(sp); + let expected: Vec<_> = expected.into_iter().map(Into::into).collect(); + let actual: Vec<_> = s.iter().copied().collect(); + assert_eq!(actual, expected, "test failed for {drop_keep:?}"); } let test_inputs = [1, 2, 3, 4, 5, 6]; diff --git a/crates/wasmi/src/engine/stack/values/vref.rs b/crates/wasmi/src/engine/stack/values/vref.rs deleted file mode 100644 index 335e9bf130..0000000000 --- a/crates/wasmi/src/engine/stack/values/vref.rs +++ /dev/null @@ -1,318 +0,0 @@ -use super::ValueStack; -use crate::{ - core::{TrapCode, UntypedValue}, - engine::DropKeep, -}; -use core::fmt; - -/// A mutable view over the [`ValueStack`]. -/// -/// This allows for a more efficient access to the [`ValueStack`] during execution. -pub struct ValueStackRef<'a> { - pub(super) stack_ptr: usize, - pub(super) values: &'a mut [UntypedValue], - /// The original stack pointer required to keep in sync. - orig_sp: &'a mut usize, -} - -impl<'a> fmt::Debug for ValueStackRef<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", &self.values[0..self.stack_ptr]) - } -} - -impl<'a> ValueStackRef<'a> { - /// Creates a new [`ValueStackRef`] from the given [`ValueStack`]. - /// - /// This also returns an exclusive reference to the stack pointer of - /// the underlying [`ValueStack`]. This is important in order to synchronize - /// the [`ValueStack`] with the changes done to the [`ValueStackRef`] - /// when necessary. - pub fn new(stack: &'a mut ValueStack) -> Self { - let sp = &mut stack.stack_ptr; - let stack_ptr = *sp; - Self { - stack_ptr, - values: &mut stack.entries[..], - orig_sp: sp, - } - } - - /// Synchronizes the original value stack pointer. - pub fn sync(&mut self) { - *self.orig_sp = self.stack_ptr; - } - - /// Returns the current capacity of the underlying [`ValueStack`]. - fn capacity(&self) -> usize { - self.values.len() - } - - /// Returns the [`UntypedValue`] at the given `index`. - /// - /// # Note - /// - /// This is an optimized convenience method that only asserts - /// that the index is within bounds in `debug` mode. - /// - /// # Safety - /// - /// This is safe since all wasmi bytecode has been validated - /// during translation and therefore cannot result in out of - /// bounds accesses. - /// - /// # Panics (Debug) - /// - /// If the `index` is out of bounds. - fn get_release_unchecked(&self, index: usize) -> UntypedValue { - debug_assert!(index < self.capacity()); - // Safety: This is safe since all wasmi bytecode has been validated - // during translation and therefore cannot result in out of - // bounds accesses. - unsafe { *self.values.get_unchecked(index) } - } - - /// Returns the [`UntypedValue`] at the given `index`. - /// - /// # Note - /// - /// This is an optimized convenience method that only asserts - /// that the index is within bounds in `debug` mode. - /// - /// # Safety - /// - /// This is safe since all wasmi bytecode has been validated - /// during translation and therefore cannot result in out of - /// bounds accesses. - /// - /// # Panics (Debug) - /// - /// If the `index` is out of bounds. - fn get_release_unchecked_mut(&mut self, index: usize) -> &mut UntypedValue { - debug_assert!(index < self.capacity()); - // Safety: This is safe since all wasmi bytecode has been validated - // during translation and therefore cannot result in out of - // bounds accesses. - unsafe { self.values.get_unchecked_mut(index) } - } - - /// Drops some amount of entries and keeps some amount of them at the new top. - /// - /// # Note - /// - /// For an amount of entries to keep `k` and an amount of entries to drop `d` - /// this has the following effect on stack `s` and stack pointer `sp`. - /// - /// 1) Copy `k` elements from indices starting at `sp - k` to `sp - k - d`. - /// 2) Adjust stack pointer: `sp -= d` - /// - /// After this operation the value stack will have `d` fewer entries and the - /// top `k` entries are the top `k` entries before this operation. - /// - /// Note that `k + d` cannot be greater than the stack length. - pub fn drop_keep(&mut self, drop_keep: DropKeep) { - let drop = drop_keep.drop(); - if drop == 0 { - // Nothing to do in this case. - return; - } - let keep = drop_keep.keep(); - if keep == 0 { - // Bail out early when there are no values to keep. - } else if keep == 1 { - // Bail out early when there is only one value to copy. - *self.get_release_unchecked_mut(self.stack_ptr - 1 - drop) = - self.get_release_unchecked(self.stack_ptr - 1); - } else { - // Copy kept values over to their new place on the stack. - // Note: We cannot use `memcpy` since the slices may overlap. - let src = self.stack_ptr - keep; - let dst = self.stack_ptr - keep - drop; - for i in 0..keep { - *self.get_release_unchecked_mut(dst + i) = self.get_release_unchecked(src + i); - } - } - self.stack_ptr -= drop; - } - - /// Returns the last stack entry of the [`ValueStackRef`]. - /// - /// # Note - /// - /// This has the same effect as [`ValueStackRef::peek`]`(1)`. - #[inline] - pub fn last(&self) -> UntypedValue { - self.get_release_unchecked(self.stack_ptr - 1) - } - - /// Returns the last stack entry of the [`ValueStack`]. - /// - /// # Note - /// - /// This has the same effect as [`ValueStackRef::peek`]`(1)`. - #[inline] - pub fn last_mut(&mut self) -> &mut UntypedValue { - self.get_release_unchecked_mut(self.stack_ptr - 1) - } - - /// Peeks the entry at the given depth from the last entry. - /// - /// # Note - /// - /// Given a `depth` of 1 has the same effect as [`ValueStackRef::last`]. - /// - /// A `depth` of 0 is invalid and undefined. - #[inline] - pub fn peek(&self, depth: usize) -> UntypedValue { - self.get_release_unchecked(self.stack_ptr - depth) - } - - /// Peeks the `&mut` entry at the given depth from the last entry. - /// - /// # Note - /// - /// Given a `depth` of 1 has the same effect as [`ValueStackRef::last_mut`]. - /// - /// A `depth` of 0 is invalid and undefined. - #[inline] - pub fn peek_mut(&mut self, depth: usize) -> &mut UntypedValue { - self.get_release_unchecked_mut(self.stack_ptr - depth) - } - - /// Pops the last [`UntypedValue`] from the [`ValueStack`]. - /// - /// # Note - /// - /// This operation heavily relies on the prior validation of - /// the executed WebAssembly bytecode for correctness. - #[inline] - pub fn pop(&mut self) -> UntypedValue { - self.stack_ptr -= 1; - self.get_release_unchecked(self.stack_ptr) - } - - /// Pops the last [`UntypedValue`] from the [`ValueStack`] as `T`. - #[inline] - pub fn pop_as(&mut self) -> T - where - T: From, - { - T::from(self.pop()) - } - - /// Pops the last pair of [`UntypedValue`] from the [`ValueStack`]. - /// - /// # Note - /// - /// - This operation is slightly more efficient than using - /// [`ValueStackRef::pop`] twice. - /// - This operation heavily relies on the prior validation of - /// the executed WebAssembly bytecode for correctness. - #[inline] - pub fn pop2(&mut self) -> (UntypedValue, UntypedValue) { - self.stack_ptr -= 2; - ( - self.get_release_unchecked(self.stack_ptr), - self.get_release_unchecked(self.stack_ptr + 1), - ) - } - - /// Pops the last triple of [`UntypedValue`] from the [`ValueStack`]. - /// - /// # Note - /// - /// - This operation is slightly more efficient than using - /// [`ValueStackRef::pop`] trice. - /// - This operation heavily relies on the prior validation of - /// the executed WebAssembly bytecode for correctness. - #[inline] - pub fn pop3(&mut self) -> (UntypedValue, UntypedValue, UntypedValue) { - self.stack_ptr -= 3; - ( - self.get_release_unchecked(self.stack_ptr), - self.get_release_unchecked(self.stack_ptr + 1), - self.get_release_unchecked(self.stack_ptr + 2), - ) - } - - /// Evaluates the given closure `f` for the 3 top most stack values. - #[inline] - pub fn eval_top3(&mut self, f: F) - where - F: FnOnce(UntypedValue, UntypedValue, UntypedValue) -> UntypedValue, - { - let (e2, e3) = self.pop2(); - let e1 = self.last(); - *self.last_mut() = f(e1, e2, e3) - } - - /// Evaluates the given closure `f` for the top most stack value. - #[inline] - pub fn eval_top(&mut self, f: F) - where - F: FnOnce(UntypedValue) -> UntypedValue, - { - let top = self.last(); - *self.last_mut() = f(top); - } - - /// Evaluates the given fallible closure `f` for the top most stack value. - /// - /// # Errors - /// - /// If the closure execution fails. - #[inline] - pub fn try_eval_top(&mut self, f: F) -> Result<(), TrapCode> - where - F: FnOnce(UntypedValue) -> Result, - { - let top = self.last(); - *self.last_mut() = f(top)?; - Ok(()) - } - - /// Evaluates the given closure `f` for the 2 top most stack values. - #[inline] - pub fn eval_top2(&mut self, f: F) - where - F: FnOnce(UntypedValue, UntypedValue) -> UntypedValue, - { - let rhs = self.pop(); - let lhs = self.last(); - *self.last_mut() = f(lhs, rhs); - } - - /// Evaluates the given fallible closure `f` for the 2 top most stack values. - /// - /// # Errors - /// - /// If the closure execution fails. - #[inline] - pub fn try_eval_top2(&mut self, f: F) -> Result<(), TrapCode> - where - F: FnOnce(UntypedValue, UntypedValue) -> Result, - { - let rhs = self.pop(); - let lhs = self.last(); - *self.last_mut() = f(lhs, rhs)?; - Ok(()) - } - - /// Pushes the [`UntypedValue`] to the end of the [`ValueStack`]. - /// - /// # Note - /// - /// - This operation heavily relies on the prior validation of - /// the executed WebAssembly bytecode for correctness. - /// - Especially the stack-depth analysis during compilation with - /// a manual stack extension before function call prevents this - /// procedure from panicking. - #[inline] - pub fn push(&mut self, entry: T) - where - T: Into, - { - *self.get_release_unchecked_mut(self.stack_ptr) = entry.into(); - self.stack_ptr += 1; - } -}