Pooling allocator: add a reuse-affinity policy.

This policy attempts to reuse the same instance slot for subsequent
instantiations of the same module. This is particularly useful when
using a pooling backend such as memfd that benefits from this reuse: for
example, in the memfd case, instantiating the same module into the same
slot allows us to avoid several calls to mmap() because the same
mappings can be reused.

The policy tracks a freelist per "compiled module ID", and when
allocating a slot for an instance, tries these three options in order:

1. A slot from the freelist for this module (i.e., last used for another
   instantiation of this particular module), or
3. A slot that was last used by some other module or never before.

The "victim" slot for choice 2 is randomly chosen.

The data structures are carefully designed so that all updates are O(1),
and there is no retry-loop in any of the random selection.

This policy is now the default when the memfd backend is selected via
the `memfd-allocator` feature flag.

crates/runtime/src/instance.rs

@@ -11,7 +11,7 @@ use crate::vmcontext::{
     VMCallerCheckedAnyfunc, VMContext, VMFunctionImport, VMGlobalDefinition, VMGlobalImport,
     VMInterrupts, VMMemoryDefinition, VMMemoryImport, VMTableDefinition, VMTableImport,
 };
-use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable, Store};
+use crate::{CompiledModuleId, ExportFunction, ExportGlobal, ExportMemory, ExportTable, Store};
 use anyhow::Error;
 use memoffset::offset_of;
 use more_asserts::assert_lt;
@@ -54,6 +54,9 @@ pub(crate) struct Instance {
     /// The `Module` this `Instance` was instantiated from.
     module: Arc<Module>,
 
+    /// The unique ID for the `Module` this `Instance` was instantiated from.
+    unique_id: Option<CompiledModuleId>,
+
     /// Offsets in the `vmctx` region, precomputed from the `module` above.
     offsets: VMOffsets<HostPtr>,
 
@@ -100,13 +103,15 @@ impl Instance {
     /// Helper for allocators; not a public API.
     pub(crate) fn create_raw(
         module: &Arc<Module>,
+        unique_id: Option<CompiledModuleId>,
         wasm_data: &'static [u8],
         memories: PrimaryMap<DefinedMemoryIndex, Memory>,
         tables: PrimaryMap<DefinedTableIndex, Table>,
         host_state: Box<dyn Any + Send + Sync>,
     ) -> Instance {
         Instance {
             module: module.clone(),
+            unique_id,
             offsets: VMOffsets::new(HostPtr, &module),
             memories,
             tables,

crates/runtime/src/instance/allocator.rs

@@ -7,7 +7,7 @@ use crate::vmcontext::{
     VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMGlobalDefinition, VMSharedSignatureIndex,
 };
 use crate::ModuleMemFds;
-use crate::Store;
+use crate::{CompiledModuleId, Store};
 use anyhow::Result;
 use std::alloc;
 use std::any::Any;
@@ -35,6 +35,9 @@ pub struct InstanceAllocationRequest<'a> {
     /// The module being instantiated.
     pub module: Arc<Module>,
 
+    /// The unique ID of the module being allocated within this engine.
+    pub unique_id: Option<CompiledModuleId>,
+
     /// The base address of where JIT functions are located.
     pub image_base: usize,
 
@@ -728,8 +731,14 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator {
         let host_state = std::mem::replace(&mut req.host_state, Box::new(()));
 
         let mut handle = {
-            let instance =
-                Instance::create_raw(&req.module, &*req.wasm_data, memories, tables, host_state);
+            let instance = Instance::create_raw(
+                &req.module,
+                req.unique_id,
+                &*req.wasm_data,
+                memories,
+                tables,
+                host_state,
+            );
             let layout = instance.alloc_layout();
             let instance_ptr = alloc::alloc(layout) as *mut Instance;
             if instance_ptr.is_null() {