diff --git a/Cargo.lock b/Cargo.lock index fc23342d8d1..252245598c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,7 +48,7 @@ dependencies = [ "cfg-if", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -1877,7 +1877,7 @@ dependencies = [ "page_size", "pkg-config", "smallvec", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -2915,6 +2915,12 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libunwind" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6639b70a7ce854b79c70d7e83f16b5dc0137cc914f3d7d03803b513ecc67ac" + [[package]] name = "libyml" version = "0.0.5" @@ -3097,6 +3103,17 @@ dependencies = [ "libc", ] +[[package]] +name = "macho-unwind-info" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4bdc8b0ce69932332cf76d24af69c3a155242af95c226b2ab6c2e371ed1149" +dependencies = [ + "thiserror 2.0.3", + "zerocopy 0.8.14", + "zerocopy-derive 0.8.14", +] + [[package]] name = "macro-wasmer-universal-test" version = "5.0.2" @@ -3669,6 +3686,48 @@ dependencies = [ "indexmap 2.6.0", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.7" @@ -3759,7 +3818,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -6793,6 +6852,7 @@ dependencies = [ "leb128", "libc", "loupe", + "macho-unwind-info", "memmap2 0.6.2", "more-asserts", "object 0.32.2", @@ -6864,12 +6924,14 @@ dependencies = [ "lazy_static", "libc", "object 0.30.4", + "phf", "rayon", "regex", "rustc_version 0.4.1", "semver 1.0.23", "smallvec", "target-lexicon 0.12.16", + "tracing", "wasmer-compiler", "wasmer-types", "wasmer-vm", @@ -7116,6 +7178,7 @@ dependencies = [ "indexmap 2.6.0", "lazy_static", "libc", + "libunwind", "loupe", "mach2", "memoffset 0.9.1", diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index f783e80620e..80f954b18a2 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -425,6 +425,7 @@ impl Compiler for CraneliftCompiler { function_call_trampolines, dynamic_function_trampolines, debug: dwarf, + got: None, }) } } diff --git a/lib/compiler-llvm/Cargo.toml b/lib/compiler-llvm/Cargo.toml index 3b693b47c66..30744268a05 100644 --- a/lib/compiler-llvm/Cargo.toml +++ b/lib/compiler-llvm/Cargo.toml @@ -15,7 +15,8 @@ version.workspace = true [dependencies] wasmer-compiler = { path = "../compiler", version = "=5.0.2", features = [ - "translator", "compiler" + "translator", + "compiler", ] } wasmer-vm = { path = "../vm", version = "=5.0.2" } wasmer-types = { path = "../types", version = "=5.0.2" } @@ -26,6 +27,8 @@ libc.workspace = true byteorder = "1" itertools = "0.10" rayon = "1.5" +phf = { version = "0.11.2", features = ["macros"] } +tracing = { version = "0.1", default-features = false, features = ["log"] } [dependencies.inkwell] package = "inkwell" diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 12fd819bdac..bcc58f9d37c 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -9,6 +9,7 @@ use inkwell::targets::FileType; use inkwell::DLLStorageClass; use rayon::iter::ParallelBridge; use rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; +use std::collections::HashSet; use std::sync::Arc; use wasmer_compiler::types::function::{Compilation, Dwarf}; use wasmer_compiler::types::module::CompileModuleInfo; @@ -23,6 +24,7 @@ use wasmer_compiler::{ }; use wasmer_types::entity::{EntityRef, PrimaryMap}; use wasmer_types::{CompileError, FunctionIndex, LocalFunctionIndex, SignatureIndex}; +use wasmer_vm::libcalls::function_pointer; //use std::sync::Mutex; @@ -90,6 +92,7 @@ impl LLVMCompiler { function_body_inputs: &PrimaryMap>, symbol_registry: &dyn SymbolRegistry, wasmer_metadata: &[u8], + binary_format: target_lexicon::BinaryFormat, ) -> Result, CompileError> { let target_machine = self.config().target_machine(target); let ctx = Context::create(); @@ -99,7 +102,7 @@ impl LLVMCompiler { let merged_bitcode = function_body_inputs.into_iter().par_bridge().map_init( || { let target_machine = self.config().target_machine(target); - FuncTranslator::new(target_machine) + FuncTranslator::new(target_machine, binary_format).unwrap() }, |func_translator, (i, input)| { let module = func_translator.translate_to_module( @@ -112,6 +115,7 @@ impl LLVMCompiler { &compile_info.table_styles, symbol_registry, )?; + Ok(module.write_bitcode_to_memory().as_slice().to_vec()) }, ); @@ -119,7 +123,7 @@ impl LLVMCompiler { let trampolines_bitcode = compile_info.module.signatures.iter().par_bridge().map_init( || { let target_machine = self.config().target_machine(target); - FuncTrampoline::new(target_machine) + FuncTrampoline::new(target_machine, binary_format).unwrap() }, |func_trampoline, (i, sig)| { let name = symbol_registry.symbol_to_name(Symbol::FunctionCallTrampoline(i)); @@ -133,7 +137,7 @@ impl LLVMCompiler { || { let target_machine = self.config().target_machine(target); ( - FuncTrampoline::new(target_machine), + FuncTrampoline::new(target_machine, binary_format).unwrap(), &compile_info.module.signatures, ) }, @@ -196,6 +200,7 @@ impl LLVMCompiler { callbacks.obj_memory_buffer(&CompiledKind::Module, &memory_buffer); } + tracing::trace!("Finished compling the module!"); Ok(memory_buffer.as_slice().to_vec()) } } @@ -228,6 +233,7 @@ impl Compiler for LLVMCompiler { function_body_inputs, symbol_registry, wasmer_metadata, + self.config.target_binary_format(target), )) } @@ -241,16 +247,25 @@ impl Compiler for LLVMCompiler { function_body_inputs: PrimaryMap>, ) -> Result { //let data = Arc::new(Mutex::new(0)); + let memory_styles = &compile_info.memory_styles; let table_styles = &compile_info.table_styles; + let binary_format = self.config.target_binary_format(target); let module = &compile_info.module; // TODO: merge constants in sections. let mut module_custom_sections = PrimaryMap::new(); - let mut frame_section_bytes = vec![]; - let mut frame_section_relocations = vec![]; + + let mut eh_frame_section_bytes = vec![]; + let mut eh_frame_section_relocations = vec![]; + + let mut compact_unwind_section_bytes = vec![]; + let mut compact_unwind_section_relocations = vec![]; + + let mut got_targets = HashSet::new(); + let functions = function_body_inputs .iter() .collect::)>>() @@ -258,11 +273,12 @@ impl Compiler for LLVMCompiler { .map_init( || { let target_machine = self.config().target_machine(target); - FuncTranslator::new(target_machine) + FuncTranslator::new(target_machine, binary_format).unwrap() }, |func_translator, (i, input)| { // TODO: remove (to serialize) //let _data = data.lock().unwrap(); + func_translator.translate( module, module_translation, @@ -289,16 +305,49 @@ impl Compiler for LLVMCompiler { ) } } + + compiled_function + .compiled_function + .relocations + .iter() + .filter(|v| v.kind.needs_got()) + .for_each(|v| _ = got_targets.insert(v.reloc_target.clone())); + + compiled_function + .custom_sections + .iter() + .map(|v| v.1.relocations.iter()) + .flatten() + .filter(|v| v.kind.needs_got()) + .for_each(|v| _ = got_targets.insert(v.reloc_target.clone())); + if compiled_function .eh_frame_section_indices .contains(§ion_index) { - let offset = frame_section_bytes.len() as u32; + let offset = eh_frame_section_bytes.len() as u32; for reloc in &mut custom_section.relocations { reloc.offset += offset; } - frame_section_bytes.extend_from_slice(custom_section.bytes.as_slice()); - frame_section_relocations.extend(custom_section.relocations); + eh_frame_section_bytes.extend_from_slice(custom_section.bytes.as_slice()); + eh_frame_section_relocations.extend(custom_section.relocations); + // TODO: we do this to keep the count right, remove it. + module_custom_sections.push(CustomSection { + protection: CustomSectionProtection::Read, + bytes: SectionBody::new_with_vec(vec![]), + relocations: vec![], + }); + } else if compiled_function + .compact_unwind_section_indices + .contains(§ion_index) + { + let offset = compact_unwind_section_bytes.len() as u32; + for reloc in &mut custom_section.relocations { + reloc.offset += offset; + } + compact_unwind_section_bytes + .extend_from_slice(custom_section.bytes.as_slice()); + compact_unwind_section_relocations.extend(custom_section.relocations); // TODO: we do this to keep the count right, remove it. module_custom_sections.push(CustomSection { protection: CustomSectionProtection::Read, @@ -320,23 +369,41 @@ impl Compiler for LLVMCompiler { }) .collect::>(); - let dwarf = if !frame_section_bytes.is_empty() { - let dwarf = Some(Dwarf::new(SectionIndex::from_u32( - module_custom_sections.len() as u32, - ))); + let debug = if !eh_frame_section_bytes.is_empty() { + let debug = Dwarf::new(SectionIndex::from_u32(module_custom_sections.len() as u32)); // Do not terminate dwarf info with a zero-length CIE. // Because more info will be added later // in lib/object/src/module.rs emit_compilation module_custom_sections.push(CustomSection { protection: CustomSectionProtection::Read, - bytes: SectionBody::new_with_vec(frame_section_bytes), - relocations: frame_section_relocations, + bytes: SectionBody::new_with_vec(eh_frame_section_bytes), + relocations: eh_frame_section_relocations, }); - dwarf + + Some(debug) } else { None }; + //println!("compact_unwind_bytes: {compact_unwind_section_bytes:?}"); + + let debug = if !compact_unwind_section_bytes.is_empty() { + let section_idx = SectionIndex::from_u32(module_custom_sections.len() as u32); + module_custom_sections.push(CustomSection { + protection: CustomSectionProtection::Read, + bytes: SectionBody::new_with_vec(compact_unwind_section_bytes), + relocations: compact_unwind_section_relocations, + }); + if let Some(mut dbg) = debug { + dbg.compact_unwind = Some(section_idx); + Some(dbg) + } else { + Some(Dwarf::new_cu(section_idx)) + } + } else { + debug + }; + let function_call_trampolines = module .signatures .values() @@ -345,7 +412,7 @@ impl Compiler for LLVMCompiler { .map_init( || { let target_machine = self.config().target_machine(target); - FuncTrampoline::new(target_machine) + FuncTrampoline::new(target_machine, binary_format).unwrap() }, |func_trampoline, sig| func_trampoline.trampoline(sig, self.config(), ""), ) @@ -360,7 +427,7 @@ impl Compiler for LLVMCompiler { .map_init( || { let target_machine = self.config().target_machine(target); - FuncTrampoline::new(target_machine) + FuncTrampoline::new(target_machine, binary_format).unwrap() }, |func_trampoline, func_type| { func_trampoline.dynamic_trampoline(func_type, self.config(), "") @@ -370,12 +437,43 @@ impl Compiler for LLVMCompiler { .into_iter() .collect::>(); + let got = if !got_targets.is_empty() { + let mut got_data = vec![]; + let mut map = std::collections::HashMap::new(); + for (i, reloc) in got_targets.into_iter().enumerate() { + match reloc { + RelocationTarget::LibCall(c) => got_data.push(function_pointer(c)), + RelocationTarget::LocalFunc(_) => todo!(), + RelocationTarget::CustomSection(_) => todo!(), + } + + map.insert(reloc, i); + } + let got_data: Vec = got_data + .into_iter() + .map(|v| v.to_ne_bytes()) + .flatten() + .collect(); + let index = SectionIndex::from_u32(module_custom_sections.len() as u32); + module_custom_sections.push(CustomSection { + protection: CustomSectionProtection::Read, + bytes: SectionBody::new_with_vec(got_data), + relocations: vec![], + }); + + Some(wasmer_compiler::types::function::GOT { index, map }) + } else { + None + }; + + tracing::trace!("Finished compling the module!"); Ok(Compilation { functions, custom_sections: module_custom_sections, function_call_trampolines, dynamic_function_trampolines, - debug: dwarf, + debug, + got, }) } } diff --git a/lib/compiler-llvm/src/object_file.rs b/lib/compiler-llvm/src/object_file.rs index dbe921e4a85..6367f34d4b7 100644 --- a/lib/compiler-llvm/src/object_file.rs +++ b/lib/compiler-llvm/src/object_file.rs @@ -1,7 +1,9 @@ use object::{Object, ObjectSection, ObjectSymbol}; +use target_lexicon::BinaryFormat; use std::collections::{HashMap, HashSet}; use std::convert::TryInto; +//use std::io::Write; use std::num::TryFromIntError; use wasmer_types::{entity::PrimaryMap, CompileError, SourceLoc}; @@ -23,114 +25,160 @@ fn map_object_err(error: object::read::Error) -> CompileError { CompileError::Codegen(format!("error parsing object file: {}", error)) } +#[derive(Debug)] pub struct CompiledFunction { pub compiled_function: wasmer_compiler::types::function::CompiledFunction, pub custom_sections: CustomSections, pub eh_frame_section_indices: Vec, + pub compact_unwind_section_indices: Vec, } +static LIBCALLS_ELF: phf::Map<&'static str, LibCall> = phf::phf_map! { + "ceilf" => LibCall::CeilF32, + "ceil" => LibCall::CeilF64, + "floorf" => LibCall::FloorF32, + "floor" => LibCall::FloorF64, + "nearbyintf" => LibCall::NearestF32, + "nearbyint" => LibCall::NearestF64, + "truncf" => LibCall::TruncF32, + "trunc" => LibCall::TruncF64, + "wasmer_vm_f32_ceil" => LibCall::CeilF32, + "wasmer_vm_f64_ceil" => LibCall::CeilF64, + "wasmer_vm_f32_floor" => LibCall::FloorF32, + "wasmer_vm_f64_floor" => LibCall::FloorF64, + "wasmer_vm_f32_nearest" => LibCall::NearestF32, + "wasmer_vm_f64_nearest" => LibCall::NearestF64, + "wasmer_vm_f32_trunc" => LibCall::TruncF32, + "wasmer_vm_f64_trunc" => LibCall::TruncF64, + "wasmer_vm_memory32_size" => LibCall::Memory32Size, + "wasmer_vm_imported_memory32_size" => LibCall::ImportedMemory32Size, + "wasmer_vm_table_copy" => LibCall::TableCopy, + "wasmer_vm_table_init" => LibCall::TableInit, + "wasmer_vm_table_fill" => LibCall::TableFill, + "wasmer_vm_table_size" => LibCall::TableSize, + "wasmer_vm_imported_table_size" => LibCall::ImportedTableSize, + "wasmer_vm_table_get" => LibCall::TableGet, + "wasmer_vm_imported_table_get" => LibCall::ImportedTableGet, + "wasmer_vm_table_set" => LibCall::TableSet, + "wasmer_vm_imported_table_set" => LibCall::ImportedTableSet, + "wasmer_vm_table_grow" => LibCall::TableGrow, + "wasmer_vm_imported_table_grow" => LibCall::ImportedTableGrow, + "wasmer_vm_func_ref" => LibCall::FuncRef, + "wasmer_vm_elem_drop" => LibCall::ElemDrop, + "wasmer_vm_memory32_copy" => LibCall::Memory32Copy, + "wasmer_vm_imported_memory32_copy" => LibCall::ImportedMemory32Copy, + "wasmer_vm_memory32_fill" => LibCall::Memory32Fill, + "wasmer_vm_imported_memory32_fill" => LibCall::ImportedMemory32Fill, + "wasmer_vm_memory32_init" => LibCall::Memory32Init, + "wasmer_vm_data_drop" => LibCall::DataDrop, + "wasmer_vm_raise_trap" => LibCall::RaiseTrap, + "wasmer_vm_memory32_atomic_wait32" => LibCall::Memory32AtomicWait32, + "wasmer_vm_imported_memory32_atomic_wait32" => LibCall::ImportedMemory32AtomicWait32, + "wasmer_vm_memory32_atomic_wait64" => LibCall::Memory32AtomicWait64, + "wasmer_vm_imported_memory32_atomic_wait64" => LibCall::ImportedMemory32AtomicWait64, + "wasmer_vm_memory32_atomic_notify" => LibCall::Memory32AtomicNotify, + "wasmer_vm_imported_memory32_atomic_notify" => LibCall::ImportedMemory32AtomicNotify, + "wasmer_vm_throw" => LibCall::Throw, + "wasmer_vm_rethrow" => LibCall::Rethrow, + "wasmer_vm_alloc_exception" => LibCall::AllocException, + "wasmer_vm_delete_exception" => LibCall::DeleteException, + "wasmer_vm_read_exception" => LibCall::ReadException, + "wasmer_vm_dbg_usize" => LibCall::DebugUsize, + "__gxx_personality_v0" => LibCall::EHPersonality, +}; + +static LIBCALLS_MACHO: phf::Map<&'static str, LibCall> = phf::phf_map! { + "_ceilf" => LibCall::CeilF32, + "_ceil" => LibCall::CeilF64, + "_floorf" => LibCall::FloorF32, + "_floor" => LibCall::FloorF64, + "_nearbyintf" => LibCall::NearestF32, + "_nearbyint" => LibCall::NearestF64, + "_truncf" => LibCall::TruncF32, + "_trunc" => LibCall::TruncF64, + "_wasmer_vm_f32_ceil" => LibCall::CeilF32, + "_wasmer_vm_f64_ceil" => LibCall::CeilF64, + "_wasmer_vm_f32_floor" => LibCall::FloorF32, + "_wasmer_vm_f64_floor" => LibCall::FloorF64, + "_wasmer_vm_f32_nearest" => LibCall::NearestF32, + "_wasmer_vm_f64_nearest" => LibCall::NearestF64, + "_wasmer_vm_f32_trunc" => LibCall::TruncF32, + "_wasmer_vm_f64_trunc" => LibCall::TruncF64, + "_wasmer_vm_memory32_size" => LibCall::Memory32Size, + "_wasmer_vm_imported_memory32_size" => LibCall::ImportedMemory32Size, + "_wasmer_vm_table_copy" => LibCall::TableCopy, + "_wasmer_vm_table_init" => LibCall::TableInit, + "_wasmer_vm_table_fill" => LibCall::TableFill, + "_wasmer_vm_table_size" => LibCall::TableSize, + "_wasmer_vm_imported_table_size" => LibCall::ImportedTableSize, + "_wasmer_vm_table_get" => LibCall::TableGet, + "_wasmer_vm_imported_table_get" => LibCall::ImportedTableGet, + "_wasmer_vm_table_set" => LibCall::TableSet, + "_wasmer_vm_imported_table_set" => LibCall::ImportedTableSet, + "_wasmer_vm_table_grow" => LibCall::TableGrow, + "_wasmer_vm_imported_table_grow" => LibCall::ImportedTableGrow, + "_wasmer_vm_func_ref" => LibCall::FuncRef, + "_wasmer_vm_elem_drop" => LibCall::ElemDrop, + "_wasmer_vm_memory32_copy" => LibCall::Memory32Copy, + "_wasmer_vm_imported_memory32_copy" => LibCall::ImportedMemory32Copy, + "_wasmer_vm_memory32_fill" => LibCall::Memory32Fill, + "_wasmer_vm_imported_memory32_fill" => LibCall::ImportedMemory32Fill, + "_wasmer_vm_memory32_init" => LibCall::Memory32Init, + "_wasmer_vm_data_drop" => LibCall::DataDrop, + "_wasmer_vm_raise_trap" => LibCall::RaiseTrap, + "_wasmer_vm_memory32_atomic_wait32" => LibCall::Memory32AtomicWait32, + "_wasmer_vm_imported_memory32_atomic_wait32" => LibCall::ImportedMemory32AtomicWait32, + "_wasmer_vm_memory32_atomic_wait64" => LibCall::Memory32AtomicWait64, + "_wasmer_vm_imported_memory32_atomic_wait64" => LibCall::ImportedMemory32AtomicWait64, + "_wasmer_vm_memory32_atomic_notify" => LibCall::Memory32AtomicNotify, + "_wasmer_vm_imported_memory32_atomic_notify" => LibCall::ImportedMemory32AtomicNotify, + "_wasmer_vm_throw" => LibCall::Throw, + "_wasmer_vm_rethrow" => LibCall::Rethrow, + "_wasmer_vm_alloc_exception" => LibCall::AllocException, + "_wasmer_vm_delete_exception" => LibCall::DeleteException, + "_wasmer_vm_read_exception" => LibCall::ReadException, + "_wasmer_vm_dbg_usize" => LibCall::DebugUsize, + "___gxx_personality_v0" => LibCall::EHPersonality, +}; + pub fn load_object_file( contents: &[u8], root_section: &str, root_section_reloc_target: RelocationTarget, mut symbol_name_to_relocation_target: F, + binary_fmt: BinaryFormat, ) -> Result where F: FnMut(&str) -> Result, CompileError>, { - // TODO: use perfect hash function? - let mut libcalls = HashMap::new(); - libcalls.insert("ceilf".to_string(), LibCall::CeilF32); - libcalls.insert("ceil".to_string(), LibCall::CeilF64); - libcalls.insert("floorf".to_string(), LibCall::FloorF32); - libcalls.insert("floor".to_string(), LibCall::FloorF64); - libcalls.insert("nearbyintf".to_string(), LibCall::NearestF32); - libcalls.insert("nearbyint".to_string(), LibCall::NearestF64); - libcalls.insert("truncf".to_string(), LibCall::TruncF32); - libcalls.insert("trunc".to_string(), LibCall::TruncF64); - libcalls.insert("wasmer_vm_f32_ceil".to_string(), LibCall::CeilF32); - libcalls.insert("wasmer_vm_f64_ceil".to_string(), LibCall::CeilF64); - libcalls.insert("wasmer_vm_f32_floor".to_string(), LibCall::FloorF32); - libcalls.insert("wasmer_vm_f64_floor".to_string(), LibCall::FloorF64); - libcalls.insert("wasmer_vm_f32_nearest".to_string(), LibCall::NearestF32); - libcalls.insert("wasmer_vm_f64_nearest".to_string(), LibCall::NearestF64); - libcalls.insert("wasmer_vm_f32_trunc".to_string(), LibCall::TruncF32); - libcalls.insert("wasmer_vm_f64_trunc".to_string(), LibCall::TruncF64); - libcalls.insert("wasmer_vm_memory32_size".to_string(), LibCall::Memory32Size); - libcalls.insert( - "wasmer_vm_imported_memory32_size".to_string(), - LibCall::ImportedMemory32Size, - ); - libcalls.insert("wasmer_vm_table_copy".to_string(), LibCall::TableCopy); - libcalls.insert("wasmer_vm_table_init".to_string(), LibCall::TableInit); - libcalls.insert("wasmer_vm_table_fill".to_string(), LibCall::TableFill); - libcalls.insert("wasmer_vm_table_size".to_string(), LibCall::TableSize); - libcalls.insert( - "wasmer_vm_imported_table_size".to_string(), - LibCall::ImportedTableSize, - ); - libcalls.insert("wasmer_vm_table_get".to_string(), LibCall::TableGet); - libcalls.insert( - "wasmer_vm_imported_table_get".to_string(), - LibCall::ImportedTableGet, - ); - libcalls.insert("wasmer_vm_table_set".to_string(), LibCall::TableSet); - libcalls.insert( - "wasmer_vm_imported_table_set".to_string(), - LibCall::ImportedTableSet, - ); - libcalls.insert("wasmer_vm_table_grow".to_string(), LibCall::TableGrow); - libcalls.insert( - "wasmer_vm_imported_table_grow".to_string(), - LibCall::ImportedTableGrow, - ); - libcalls.insert("wasmer_vm_func_ref".to_string(), LibCall::FuncRef); - libcalls.insert("wasmer_vm_elem_drop".to_string(), LibCall::ElemDrop); - libcalls.insert("wasmer_vm_memory32_copy".to_string(), LibCall::Memory32Copy); - libcalls.insert( - "wasmer_vm_imported_memory32_copy".to_string(), - LibCall::ImportedMemory32Copy, - ); - libcalls.insert("wasmer_vm_memory32_fill".to_string(), LibCall::Memory32Fill); - libcalls.insert( - "wasmer_vm_imported_memory32_fill".to_string(), - LibCall::ImportedMemory32Fill, - ); - libcalls.insert("wasmer_vm_memory32_init".to_string(), LibCall::Memory32Init); - libcalls.insert("wasmer_vm_data_drop".to_string(), LibCall::DataDrop); - libcalls.insert("wasmer_vm_raise_trap".to_string(), LibCall::RaiseTrap); - libcalls.insert( - "wasmer_vm_memory32_atomic_wait32".to_string(), - LibCall::Memory32AtomicWait32, - ); - libcalls.insert( - "wasmer_vm_imported_memory32_atomic_wait32".to_string(), - LibCall::ImportedMemory32AtomicWait32, - ); - libcalls.insert( - "wasmer_vm_memory32_atomic_wait64".to_string(), - LibCall::Memory32AtomicWait64, - ); - libcalls.insert( - "wasmer_vm_imported_memory32_atomic_wait64".to_string(), - LibCall::ImportedMemory32AtomicWait64, - ); - libcalls.insert( - "wasmer_vm_memory32_atomic_notify".to_string(), - LibCall::Memory32AtomicNotify, - ); - libcalls.insert( - "wasmer_vm_imported_memory32_atomic_notify".to_string(), - LibCall::ImportedMemory32AtomicNotify, - ); - - let elf = object::File::parse(contents).map_err(map_object_err)?; + // -- Uncomment to enable dumping intermediate LLVM objects + //let mut fs = std::fs::OpenOptions::new() + // .write(true) + // .create(true) + // .open(format!( + // "{}/obj_{root_section}.o", + // std::env!("LLVM_EH_TESTS_DUMP_DIR") + // )) + // .unwrap(); + //fs.write_all(contents).unwrap(); + + let obj = object::File::parse(contents).map_err(map_object_err)?; + + let libcalls = match binary_fmt { + BinaryFormat::Elf => &LIBCALLS_ELF, + BinaryFormat::Macho => &LIBCALLS_MACHO, + _ => { + return Err(CompileError::UnsupportedTarget(format!( + "Unsupported binary format {binary_fmt:?}" + ))) + } + }; let mut visited: HashSet = HashSet::new(); let mut worklist: Vec = Vec::new(); let mut section_targets: HashMap = HashMap::new(); - let root_section_index = elf + let root_section_index = obj .section_by_name(root_section) .ok_or_else(|| CompileError::Codegen(format!("no section named {}", root_section)))? .index(); @@ -140,6 +188,7 @@ where section_targets.insert(root_section_index, root_section_reloc_target); let mut next_custom_section: u32 = 0; + let mut elf_section_to_target = |elf_section_index: object::read::SectionIndex| { *section_targets.entry(elf_section_index).or_insert_with(|| { let next = SectionIndex::from_u32(next_custom_section); @@ -155,7 +204,7 @@ where let mut relocations: HashMap> = HashMap::new(); // Each iteration of this loop pulls a section and the relocations - // relocations that apply to it. We begin with the ".root_section" + // that apply to it. We begin with the ".root_section" // section, and then parse all relocation sections that apply to that // section. Those relocations may refer to additional sections which we // then add to the worklist until we've visited the closure of @@ -170,27 +219,165 @@ where worklist.push(root_section_index); visited.insert(root_section_index); - // Also add any .eh_frame sections. + // Add any .eh_frame sections. let mut eh_frame_section_indices = vec![]; - for section in elf.sections() { - if section.kind() == object::SectionKind::Elf(object::elf::SHT_X86_64_UNWIND) { - let index = section.index(); + + // Add macos-specific unwind sections. + let mut compact_unwind_section_indices = vec![]; + + for section in obj.sections() { + let index = section.index(); + //println!( + // "section kind: {:?}, section name: {:?}", + // section.kind(), + // section.name() + //); + if section.kind() == object::SectionKind::Elf(object::elf::SHT_X86_64_UNWIND) + || section.name().unwrap_or_default() == "__eh_frame" + { worklist.push(index); - visited.insert(index); + + //visited.insert(index); eh_frame_section_indices.push(index); // This allocates a custom section index for the ELF section. elf_section_to_target(index); + } else if section.name().unwrap_or_default() == "__compact_unwind" { + worklist.push(index); + compact_unwind_section_indices.push(index); + elf_section_to_target(index); } + // } else if section.name().unwrap_or_default() == "__gcc_except_tab" + // || section.name().unwrap_or_default() == ".gcc_except_tab" + // { + // worklist.push(index); + // elf_section_to_target(index); + // } else if section.name().unwrap_or_default() == "__wasmer_eh_type_info" { + // worklist.push(index); + // visited.insert(index); + // eh_frame_section_indices.push(index); + // elf_section_to_target(index); + // } + + //if section.name().unwrap_or_default() == "__got" { + // got_index = Some(section.index()); + // worklist.push(index); + // elf_section_to_target(index); + //} } while let Some(section_index) = worklist.pop() { - for (offset, reloc) in elf + let sec = obj .section_by_index(section_index) - .map_err(map_object_err)? - .relocations() - { - let kind = match (elf.architecture(), reloc.kind(), reloc.size()) { + .map_err(map_object_err)?; + let relocs = sec.relocations(); + //println!( + // "analysing section: {:?}; it has relocs: {:?}", + // sec.name(), + // sec.relocations().into_iter().collect::>() + //); + + //if sec.name().unwrap_or_default() == "__wasmer_eh_type_info" { + // eprintln!("eh: {:?}", sec.data()); + //} + for (offset, reloc) in relocs { + let mut addend = reloc.addend(); + let target = match reloc.target() { + object::read::RelocationTarget::Symbol(index) => { + let symbol = obj.symbol_by_index(index).map_err(map_object_err)?; + let symbol_name = symbol.name().map_err(map_object_err)?; + if symbol.kind() == object::SymbolKind::Section { + match symbol.section() { + object::SymbolSection::Section(section_index) => { + if section_index == root_section_index { + root_section_reloc_target + } else { + if visited.insert(section_index) { + //let name = obj.section_by_index(section_index); + //let name = name.unwrap(); + //let name = name.name().unwrap_or_default(); + //println!("Adding section: {:?}", name); + worklist.push(section_index); + + //if name == ".gcc_except_table" + // || name == "__wasmer_eh_type_info" + //{ + // //eh_frame_section_indices.push(section_index); + //} + } + elf_section_to_target(section_index) + } + } + _ => { + return Err(CompileError::Codegen(format!( + "relocation targets unknown section {:?}", + reloc + ))); + } + } + // Maybe a libcall then? + } else if let Some(libcall) = libcalls.get(symbol_name) { + RelocationTarget::LibCall(*libcall) + } else if let Ok(Some(reloc_target)) = + symbol_name_to_relocation_target(symbol_name) + { + reloc_target + } else if let object::SymbolSection::Section(section_index) = symbol.section() { + // TODO: Encode symbol address into addend, I think this is a bit hacky. + addend = addend.wrapping_add(symbol.address() as i64); + + if section_index == root_section_index { + root_section_reloc_target + } else { + if visited.insert(section_index) { + worklist.push(section_index); + } + + elf_section_to_target(section_index) + } + } else { + return Err(CompileError::Codegen(format!( + "relocation {reloc:?} targets unknown symbol '{symbol:?}'", + ))); + } + } + + object::read::RelocationTarget::Section(index) => { + if index == root_section_index { + root_section_reloc_target + } else { + if visited.insert(index) { + worklist.push(index); + } + elf_section_to_target(index) + } + } + + object::read::RelocationTarget::Absolute => { + // Wasm-produced object files should never have absolute + // addresses in them because none of the parts of the Wasm + // VM, nor the generated code are loaded at fixed addresses. + return Err(CompileError::Codegen(format!( + "relocation targets absolute address {:?}", + reloc + ))); + } + + // `object::read::RelocationTarget` is a + // non-exhaustive enum (`#[non_exhaustive]`), so it + // could have additional variants added in the + // future. Therefore, when matching against variants + // of non-exhaustive enums, an extra wildcard arm must + // be added to account for any future variants. + t => { + return Err(CompileError::Codegen(format!( + "relocation target is unknown `{:?}`", + t + ))); + } + }; + let kind = match (obj.architecture(), reloc.kind(), reloc.size()) { (_, object::RelocationKind::Absolute, 64) => RelocationKind::Abs8, + (_, object::RelocationKind::Absolute, 32) => RelocationKind::Abs4, ( object::Architecture::X86_64, object::RelocationKind::Elf(object::elf::R_X86_64_PC64), @@ -279,100 +466,89 @@ where object::RelocationKind::Elf(object::elf::R_AARCH64_LDST64_ABS_LO12_NC), 0, ) => RelocationKind::Aarch64Ldst64AbsLo12Nc, - - _ => { - return Err(CompileError::Codegen(format!( - "unknown relocation {:?}", - reloc - ))); - } - }; - let mut addend = reloc.addend(); - let target = match reloc.target() { - object::read::RelocationTarget::Symbol(index) => { - let symbol = elf.symbol_by_index(index).map_err(map_object_err)?; - let symbol_name = symbol.name().map_err(map_object_err)?; - if symbol.kind() == object::SymbolKind::Section { - match symbol.section() { - object::SymbolSection::Section(section_index) => { - if section_index == root_section_index { - root_section_reloc_target - } else { - if visited.insert(section_index) { - worklist.push(section_index); - } - elf_section_to_target(section_index) - } - } - _ => { - return Err(CompileError::Codegen(format!( - "relocation targets unknown section {:?}", - reloc - ))); - } + (object::Architecture::Aarch64, object::RelocationKind::MachO { value, .. }, _) => { + match value { + object::macho::ARM64_RELOC_UNSIGNED => { + RelocationKind::MachoArm64RelocUnsigned } - // Maybe a libcall then? - } else if let Some(libcall) = libcalls.get(symbol_name) { - RelocationTarget::LibCall(*libcall) - } else if let Some(reloc_target) = - symbol_name_to_relocation_target(symbol_name)? - { - reloc_target - } else if let object::SymbolSection::Section(section_index) = symbol.section() { - // TODO: Encode symbol address into addend, I think this is a bit hacky. - addend = addend.wrapping_add(symbol.address() as i64); - - if section_index == root_section_index { - root_section_reloc_target - } else { - if visited.insert(section_index) { - worklist.push(section_index); - } - - elf_section_to_target(section_index) + object::macho::ARM64_RELOC_SUBTRACTOR => { + RelocationKind::MachoArm64RelocSubtractor + } + object::macho::ARM64_RELOC_BRANCH26 => { + RelocationKind::MachoArm64RelocBranch26 + } + object::macho::ARM64_RELOC_PAGE21 => RelocationKind::MachoArm64RelocPage21, + object::macho::ARM64_RELOC_PAGEOFF12 => { + RelocationKind::MachoArm64RelocPageoff12 + } + object::macho::ARM64_RELOC_GOT_LOAD_PAGE21 => { + RelocationKind::MachoArm64RelocGotLoadPage21 + } + object::macho::ARM64_RELOC_GOT_LOAD_PAGEOFF12 => { + RelocationKind::MachoArm64RelocGotLoadPageoff12 + } + object::macho::ARM64_RELOC_POINTER_TO_GOT => { + RelocationKind::MachoArm64RelocPointerToGot + } + object::macho::ARM64_RELOC_TLVP_LOAD_PAGE21 => { + RelocationKind::MachoArm64RelocTlvpLoadPage21 + } + object::macho::ARM64_RELOC_TLVP_LOAD_PAGEOFF12 => { + RelocationKind::MachoArm64RelocTlvpLoadPageoff12 + } + object::macho::ARM64_RELOC_ADDEND => RelocationKind::MachoArm64RelocAddend, + _ => { + return Err(CompileError::Codegen(format!( + "unknown relocation {:?}", + reloc + ))) } - } else { - return Err(CompileError::Codegen(format!( - "relocation targets unknown symbol {:?}", - reloc - ))); } } - - object::read::RelocationTarget::Section(index) => { - if index == root_section_index { - root_section_reloc_target - } else { - if visited.insert(index) { - worklist.push(index); + (object::Architecture::X86_64, object::RelocationKind::MachO { value, .. }, _) => { + match value { + object::macho::X86_64_RELOC_UNSIGNED => { + RelocationKind::MachoX86_64RelocUnsigned + } + object::macho::X86_64_RELOC_SIGNED => { + RelocationKind::MachoX86_64RelocSigned + } + object::macho::X86_64_RELOC_BRANCH => { + RelocationKind::MachoX86_64RelocBranch + } + object::macho::X86_64_RELOC_GOT_LOAD => { + RelocationKind::MachoX86_64RelocGotLoad + } + object::macho::X86_64_RELOC_GOT => RelocationKind::MachoX86_64RelocGot, + object::macho::X86_64_RELOC_SUBTRACTOR => { + RelocationKind::MachoX86_64RelocSubtractor + } + object::macho::X86_64_RELOC_SIGNED_1 => { + RelocationKind::MachoX86_64RelocSigned1 + } + object::macho::X86_64_RELOC_SIGNED_2 => { + RelocationKind::MachoX86_64RelocSigned2 + } + object::macho::X86_64_RELOC_SIGNED_4 => { + RelocationKind::MachoX86_64RelocSigned4 + } + object::macho::X86_64_RELOC_TLV => RelocationKind::MachoX86_64RelocTlv, + _ => { + return Err(CompileError::Codegen(format!( + "unknown relocation {:?}", + reloc + ))) } - elf_section_to_target(index) } } - - object::read::RelocationTarget::Absolute => { - // Wasm-produced object files should never have absolute - // addresses in them because none of the parts of the Wasm - // VM, nor the generated code are loaded at fixed addresses. + _ => { return Err(CompileError::Codegen(format!( - "relocation targets absolute address {:?}", + "unknown relocation {:?}", reloc ))); } - - // `object::read::RelocationTarget` is a - // non-exhaustive enum (`#[non_exhaustive]`), so it - // could have additional variants added in the - // future. Therefore, when matching against variants - // of non-exhaustive enums, an extra wildcard arm must - // be added to account for any future variants. - t => { - return Err(CompileError::Codegen(format!( - "relocation target is unknown `{:?}`", - t - ))); - } }; + relocations .entry(section_index) .or_default() @@ -400,6 +576,36 @@ where }) .collect::, _>>()?; + let compact_unwind_section_indices = compact_unwind_section_indices + .iter() + .map(|index| { + section_to_custom_section.get(index).map_or_else( + || { + Err(CompileError::Codegen(format!( + "_compact_unwind section with index={:?} was never loaded", + index + ))) + }, + |idx| Ok(*idx), + ) + }) + .collect::, _>>()?; + + //let got = if let Some(index) = got_index { + // Some(section_to_custom_section.get(&index).map_or_else( + // || { + // Err(CompileError::Codegen(format!( + // "got section with index={:?} was never loaded", + // index + // ))) + // }, + // |idx| Ok(*idx), + // )?) + //} else { + // None + //}; + + //println!("cus: {section_to_custom_section:?}"); let mut custom_sections = section_to_custom_section .iter() .map(|(elf_section_index, custom_section_index)| { @@ -408,7 +614,7 @@ where CustomSection { protection: CustomSectionProtection::Read, bytes: SectionBody::new_with_vec( - elf.section_by_index(*elf_section_index) + obj.section_by_index(*elf_section_index) .unwrap() .data() .unwrap() @@ -428,7 +634,7 @@ where .collect::>(); let function_body = FunctionBody { - body: elf + body: obj .section_by_index(root_section_index) .unwrap() .data() @@ -437,6 +643,10 @@ where unwind_info: None, }; + //println!("eh_frame_section_indices: {eh_frame_section_indices:?}"); + //println!("compact_unwind_section_indices: {compact_unwind_section_indices:?}"); + //println!("custom_sections: {custom_sections:#?}"); + let address_map = FunctionAddressMap { instructions: vec![InstructionAddressMap { srcloc: SourceLoc::default(), @@ -462,5 +672,6 @@ where }, custom_sections, eh_frame_section_indices, + compact_unwind_section_indices, }) } diff --git a/lib/compiler-llvm/src/trampoline/wasm.rs b/lib/compiler-llvm/src/trampoline/wasm.rs index 17c46cebf5f..515678b8ef7 100644 --- a/lib/compiler-llvm/src/trampoline/wasm.rs +++ b/lib/compiler-llvm/src/trampoline/wasm.rs @@ -16,6 +16,7 @@ use inkwell::{ AddressSpace, DLLStorageClass, }; use std::{cmp, convert::TryInto}; +use target_lexicon::BinaryFormat; use wasmer_compiler::types::{function::FunctionBody, relocation::RelocationTarget}; use wasmer_types::{CompileError, FunctionType as FuncType, LocalFunctionIndex}; @@ -23,18 +24,35 @@ pub struct FuncTrampoline { ctx: Context, target_machine: TargetMachine, abi: Box, + binary_fmt: BinaryFormat, + func_section: String, } -const FUNCTION_SECTION: &str = "__TEXT,wasmer_trmpl"; // Needs to be between 1 and 16 chars +const FUNCTION_SECTION_ELF: &str = "__TEXT,wasmer_trmpl"; // Needs to be between 1 and 16 chars +const FUNCTION_SECTION_MACHO: &str = "wasmer_trmpl"; // Needs to be between 1 and 16 chars impl FuncTrampoline { - pub fn new(target_machine: TargetMachine) -> Self { + pub fn new( + target_machine: TargetMachine, + binary_fmt: BinaryFormat, + ) -> Result { let abi = get_abi(&target_machine); - Self { + Ok(Self { ctx: Context::create(), target_machine, abi, - } + func_section: match binary_fmt { + BinaryFormat::Elf => FUNCTION_SECTION_ELF.to_string(), + BinaryFormat::Macho => FUNCTION_SECTION_MACHO.to_string(), + _ => { + return Err(CompileError::UnsupportedTarget(format!( + "Unsupported binary format: {:?}", + binary_fmt + ))) + } + }, + binary_fmt, + }) } pub fn trampoline_to_module( @@ -68,13 +86,15 @@ impl FuncTrampoline { let trampoline_func = module.add_function(name, trampoline_ty, Some(Linkage::External)); trampoline_func .as_global_value() - .set_section(Some(FUNCTION_SECTION)); + .set_section(Some(&self.func_section)); trampoline_func .as_global_value() .set_linkage(Linkage::DLLExport); trampoline_func .as_global_value() .set_dll_storage_class(DLLStorageClass::Export); + trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.uwtable); + trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer); self.generate_trampoline( trampoline_func, ty, @@ -107,6 +127,13 @@ impl FuncTrampoline { callbacks.postopt_ir(&function, &module); } + // -- Uncomment to enable dumping intermediate LLVM objects + //module + // .print_to_file(format!( + // "{}/obj_trmpl.ll", + // std::env!("LLVM_EH_TESTS_DUMP_DIR") + // )) + // .unwrap(); Ok(module) } @@ -133,9 +160,11 @@ impl FuncTrampoline { compiled_function, custom_sections, eh_frame_section_indices, + mut compact_unwind_section_indices, + .. } = load_object_file( mem_buf_slice, - FUNCTION_SECTION, + &self.func_section, RelocationTarget::LocalFunc(LocalFunctionIndex::from_u32(0)), |name: &str| { Err(CompileError::Codegen(format!( @@ -143,14 +172,16 @@ impl FuncTrampoline { name ))) }, + self.binary_fmt, )?; let mut all_sections_are_eh_sections = true; - if eh_frame_section_indices.len() != custom_sections.len() { + let mut unwind_section_indices = eh_frame_section_indices; + unwind_section_indices.append(&mut compact_unwind_section_indices); + if unwind_section_indices.len() != custom_sections.len() { all_sections_are_eh_sections = false; } else { - let mut eh_frame_section_indices = eh_frame_section_indices; - eh_frame_section_indices.sort_unstable(); - for (idx, section_idx) in eh_frame_section_indices.iter().enumerate() { + unwind_section_indices.sort_unstable(); + for (idx, section_idx) in unwind_section_indices.iter().enumerate() { if idx as u32 != section_idx.as_u32() { all_sections_are_eh_sections = false; break; @@ -200,7 +231,7 @@ impl FuncTrampoline { } trampoline_func .as_global_value() - .set_section(Some(FUNCTION_SECTION)); + .set_section(Some(&self.func_section)); trampoline_func .as_global_value() .set_linkage(Linkage::DLLExport); @@ -258,9 +289,11 @@ impl FuncTrampoline { compiled_function, custom_sections, eh_frame_section_indices, + mut compact_unwind_section_indices, + .. } = load_object_file( mem_buf_slice, - FUNCTION_SECTION, + &self.func_section, RelocationTarget::LocalFunc(LocalFunctionIndex::from_u32(0)), |name: &str| { Err(CompileError::Codegen(format!( @@ -268,14 +301,17 @@ impl FuncTrampoline { name ))) }, + self.binary_fmt, )?; let mut all_sections_are_eh_sections = true; - if eh_frame_section_indices.len() != custom_sections.len() { + let mut unwind_section_indices = eh_frame_section_indices; + unwind_section_indices.append(&mut compact_unwind_section_indices); + + if unwind_section_indices.len() != custom_sections.len() { all_sections_are_eh_sections = false; } else { - let mut eh_frame_section_indices = eh_frame_section_indices; - eh_frame_section_indices.sort_unstable(); - for (idx, section_idx) in eh_frame_section_indices.iter().enumerate() { + unwind_section_indices.sort_unstable(); + for (idx, section_idx) in unwind_section_indices.iter().enumerate() { if idx as u32 != section_idx.as_u32() { all_sections_are_eh_sections = false; break; diff --git a/lib/compiler-singlepass/src/compiler.rs b/lib/compiler-singlepass/src/compiler.rs index d5b5ae93541..82b7bdc69ee 100644 --- a/lib/compiler-singlepass/src/compiler.rs +++ b/lib/compiler-singlepass/src/compiler.rs @@ -261,6 +261,7 @@ impl Compiler for SinglepassCompiler { function_call_trampolines, dynamic_function_trampolines, debug: dwarf, + got: None, }) } diff --git a/lib/compiler/Cargo.toml b/lib/compiler/Cargo.toml index 620aee1607e..85b1f93ad7a 100644 --- a/lib/compiler/Cargo.toml +++ b/lib/compiler/Cargo.toml @@ -27,7 +27,6 @@ loupe = { version = "0.1.3", optional = true, features = [ "enable-indexmap", ] } - backtrace = "0.3" memmap2 = "0.6" more-asserts = "0.2" @@ -54,6 +53,9 @@ windows-sys = { version = "0.59", features = [ "Win32_System_Diagnostics_Debug", ] } +[target.'cfg(all(target_os = "macos", target_arch = "aarch64"))'.dependencies] +macho-unwind-info = "0.5.0" + [features] default = ["std"] # This feature is for compiler implementors, it enables using `Compiler` and diff --git a/lib/compiler/src/artifact_builders/artifact_builder.rs b/lib/compiler/src/artifact_builders/artifact_builder.rs index 3ca3483a580..38a0117988b 100644 --- a/lib/compiler/src/artifact_builders/artifact_builder.rs +++ b/lib/compiler/src/artifact_builders/artifact_builder.rs @@ -15,7 +15,7 @@ use crate::{ SerializableModule, }, types::{ - function::{CompiledFunctionFrameInfo, Dwarf, FunctionBody}, + function::{CompiledFunctionFrameInfo, Dwarf, FunctionBody, GOT}, module::CompileModuleInfo, relocation::Relocation, section::{CustomSection, SectionIndex}, @@ -143,6 +143,7 @@ impl ArtifactBuild { debug: compilation.debug, libcall_trampolines, libcall_trampoline_len, + got: compilation.got, }; let serializable = SerializableModule { compilation: serializable_compilation, @@ -203,6 +204,11 @@ impl ArtifactBuild { self.serializable.compilation.debug.as_ref() } + /// Get a reference to the [`GOT`], if available. + pub fn get_got_ref(&self) -> Option<&GOT> { + self.serializable.compilation.got.as_ref() + } + /// Get Function Relocations ref pub fn get_frame_info_ref(&self) -> &PrimaryMap { &self.serializable.compilation.function_frame_info @@ -423,6 +429,16 @@ impl ArtifactBuildFromArchive { } } + /// Get a reference to the [`GOT`], if available. + pub fn get_got_ref(&self) -> Option { + match self.cell.borrow_dependent().compilation.got { + ArchivedOption::Some(ref x) => { + Some(rkyv::deserialize::<_, rkyv::rancor::Error>(x).unwrap()) + } + ArchivedOption::None => None, + } + } + /// Get Function Relocations ref pub fn get_frame_info_ref( &self, diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index 06e81eb3683..659c2a51c40 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -11,7 +11,11 @@ use crate::{ lib::std::vec::IntoIter, register_frame_info, resolve_imports, serialize::{MetadataHeader, SerializableModule}, - types::target::{CpuFeature, Target}, + types::{ + function::GOT, + section::CustomSectionLike, + target::{CpuFeature, Target}, + }, ArtifactBuild, ArtifactBuildFromArchive, ArtifactCreate, Engine, EngineInner, Features, FrameInfosVariant, FunctionExtent, GlobalFrameInfoRegistration, InstantiationError, ModuleEnvironment, Tunables, @@ -324,6 +328,28 @@ impl Artifact { )?, }; + let got_info: Option<(usize, GOT)> = match &artifact { + ArtifactBuildVariant::Plain(ref p) => { + if let Some(got) = p.get_got_ref() { + let got = got.clone(); + let body = p.get_custom_sections_ref()[got.index].bytes.as_ptr() as usize; + Some((body, got)) + } else { + None + } + } + + ArtifactBuildVariant::Archived(ref p) => { + if let Some(got) = p.get_got_ref() { + let got = got.clone(); + let body = p.get_custom_sections_ref()[got.index].bytes().as_ptr() as usize; + Some((body, got)) + } else { + None + } + } + }; + match &artifact { ArtifactBuildVariant::Plain(p) => link_module( module_info, @@ -337,6 +363,7 @@ impl Artifact { .map(|(k, v)| (k, v.iter())), p.get_libcall_trampolines(), p.get_libcall_trampoline_len(), + got_info, ), ArtifactBuildVariant::Archived(a) => link_module( module_info, @@ -350,6 +377,7 @@ impl Artifact { .map(|(k, v)| (k, v.iter())), a.get_libcall_trampolines(), a.get_libcall_trampoline_len(), + got_info, ), }; @@ -368,17 +396,18 @@ impl Artifact { ArtifactBuildVariant::Plain(p) => p.get_debug_ref().cloned(), ArtifactBuildVariant::Archived(a) => a.get_debug_ref(), }; - let eh_frame = match debug_ref { - Some(debug) => { + + let eh_frame = match debug_ref.as_ref().and_then(|v| v.eh_frame) { + Some(eh_frame) => { let eh_frame_section_size = match &artifact { ArtifactBuildVariant::Plain(p) => { - p.get_custom_sections_ref()[debug.eh_frame].bytes.len() + p.get_custom_sections_ref()[eh_frame].bytes.len() } ArtifactBuildVariant::Archived(a) => { - a.get_custom_sections_ref()[debug.eh_frame].bytes.len() + a.get_custom_sections_ref()[eh_frame].bytes.len() } }; - let eh_frame_section_pointer = custom_sections[debug.eh_frame]; + let eh_frame_section_pointer = custom_sections[eh_frame]; Some(unsafe { std::slice::from_raw_parts(*eh_frame_section_pointer, eh_frame_section_size) }) @@ -386,9 +415,25 @@ impl Artifact { None => None, }; + let compact_unwind = match debug_ref.and_then(|v| v.compact_unwind) { + Some(cu) => { + let cu_section_size = match &artifact { + ArtifactBuildVariant::Plain(p) => p.get_custom_sections_ref()[cu].bytes.len(), + ArtifactBuildVariant::Archived(a) => { + a.get_custom_sections_ref()[cu].bytes.len() + } + }; + let cu_section_pointer = custom_sections[cu]; + Some((cu_section_pointer, cu_section_size)) + } + None => None, + }; + // Make all code compiled thus far executable. engine_inner.publish_compiled_code(); + engine_inner.publish_compact_unwind(compact_unwind)?; + engine_inner.publish_eh_frame(eh_frame)?; let finished_function_lengths = finished_functions diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index df9d58e9541..80e0fe2ac5d 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -435,7 +435,6 @@ impl EngineInner { ) }) .collect::>(); - Ok(( allocated_functions_result, allocated_function_call_trampolines, @@ -464,6 +463,23 @@ impl EngineInner { Ok(()) } + #[cfg(not(target_arch = "wasm32"))] + /// Register macos-specific exception handling information associated with the code. + pub(crate) fn publish_compact_unwind( + &mut self, + compact_unwind: Option<(SectionBodyPtr, usize)>, + ) -> Result<(), CompileError> { + self.code_memory + .last_mut() + .unwrap() + .unwind_registry_mut() + .add_compact_unwind(compact_unwind) + .map_err(|e| { + CompileError::Resource(format!("Error while publishing the unwind code: {}", e)) + })?; + Ok(()) + } + /// Shared signature registry. #[cfg(not(target_arch = "wasm32"))] pub fn signatures(&self) -> &SignatureRegistry { diff --git a/lib/compiler/src/engine/link.rs b/lib/compiler/src/engine/link.rs index 5ba13439e5d..2ee43f6fd9d 100644 --- a/lib/compiler/src/engine/link.rs +++ b/lib/compiler/src/engine/link.rs @@ -3,13 +3,14 @@ use crate::{ get_libcall_trampoline, types::{ + function::GOT, relocation::{RelocationKind, RelocationLike, RelocationTarget}, section::SectionIndex, }, FunctionExtent, }; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, ptr::{read_unaligned, write_unaligned}, }; @@ -21,21 +22,34 @@ fn apply_relocation( r: &impl RelocationLike, allocated_functions: &PrimaryMap, allocated_sections: &PrimaryMap, - libcall_trampolines: SectionIndex, + libcall_trampolines_sec_idx: SectionIndex, libcall_trampoline_len: usize, riscv_pcrel_hi20s: &mut HashMap, + got_info: &Option<(usize, GOT)>, ) { - let target_func_address: usize = match r.reloc_target() { + let reloc_target = r.reloc_target(); + let target_func_address: usize = match reloc_target { RelocationTarget::LocalFunc(index) => *allocated_functions[index].ptr as usize, RelocationTarget::LibCall(libcall) => { // Use the direct target of the libcall if the relocation supports // a full 64-bit address. Otherwise use a trampoline. - if r.kind() == RelocationKind::Abs8 || r.kind() == RelocationKind::X86PCRel8 { + if matches!( + r.kind(), + RelocationKind::Abs8 + | RelocationKind::X86PCRel8 + | RelocationKind::MachoArm64RelocUnsigned + | RelocationKind::MachoX86_64RelocUnsigned + | RelocationKind::MachoArm64RelocGotLoadPage21 + | RelocationKind::MachoArm64RelocGotLoadPageoff12 + | RelocationKind::MachoArm64RelocPointerToGot + ) { function_pointer(libcall) + //} else if matches!(r.kind(), RelocationKind::MachoArm64RelocPointerToGot) { + // Box::leak(Box::new(function_pointer(libcall))) as *mut _ as usize } else { get_libcall_trampoline( libcall, - allocated_sections[libcall_trampolines].0 as usize, + allocated_sections[libcall_trampolines_sec_idx].0 as usize, libcall_trampoline_len, ) } @@ -45,6 +59,9 @@ fn apply_relocation( } }; + // A set of addresses at which a SUBTRACTOR relocation was applied. + let mut macho_aarch64_subtractor_addresses = HashSet::new(); + match r.kind() { RelocationKind::Abs8 => unsafe { let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64); @@ -151,7 +168,7 @@ fn apply_relocation( | read_unaligned(reloc_address as *mut u32); write_unaligned(reloc_address as *mut u32, reloc_abs); }, - RelocationKind::Aarch64AdrPrelPgHi21 => unsafe { + RelocationKind::Aarch64AdrPrelPgHi21 | RelocationKind::MachoArm64RelocGotLoadPage21 => unsafe { let (reloc_address, delta) = r.for_address(body, target_func_address as u64); let delta = delta as isize; @@ -209,6 +226,50 @@ fn apply_relocation( | (read_unaligned(reloc_address as *mut u32) & 0xFFC003FF); write_unaligned(reloc_address as *mut u32, reloc_delta); }, + RelocationKind::MachoArm64RelocGotLoadPageoff12 => unsafe { + // Hacky: we replace the `ldr` instruction with an `add` instruction, as we already + // know the absolute address of a function, and we don't need any GOT. + let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64); + let ldr = read_unaligned(reloc_address as *mut u32); + let xn = ldr & 0xFFFFF; + + let new_op = 0x91000000 | (reloc_delta as u32) << 10 | (xn << 5) | xn; + write_unaligned(reloc_address as *mut u32, new_op); + }, + RelocationKind::MachoArm64RelocSubtractor | RelocationKind::MachoX86_64RelocSubtractor => unsafe { + let (reloc_address, reloc_sub) = r.for_address(body, target_func_address as u64); + macho_aarch64_subtractor_addresses.insert(reloc_address); + write_unaligned(reloc_address as *mut u64, reloc_sub); + }, + + RelocationKind::MachoArm64RelocUnsigned | RelocationKind::MachoX86_64RelocUnsigned => unsafe { + let (reloc_address, mut reloc_delta) = r.for_address(body, target_func_address as u64); + + if macho_aarch64_subtractor_addresses.contains(&reloc_address) { + reloc_delta -= read_unaligned(reloc_address as *mut u64); + } + + write_unaligned(reloc_address as *mut u64, reloc_delta); + }, + + RelocationKind::MachoArm64RelocPointerToGot => unsafe { + if let Some(got) = got_info { + let base = got.0; + let base = std::mem::transmute::(base); + + if let Some(reloc_idx) = got.1.map.get(&reloc_target) { + let got_address = base.wrapping_add(*reloc_idx); + + let (reloc_address, _reloc_delta) = r.for_address(body, got_address as u64); + + write_unaligned(reloc_address as *mut u64, got_address as u64); + } else { + panic!("Missing GOT info for reloc target {reloc_target:?}"); + } + } else { + panic!("Missing GOT info for reloc target {reloc_target:?}"); + } + }, kind => panic!( "Relocation kind unsupported in the current architecture {}", kind @@ -236,6 +297,7 @@ pub fn link_module<'a>( >, libcall_trampolines: SectionIndex, trampoline_len: usize, + got_info: Option<(usize, GOT)>, ) { let mut riscv_pcrel_hi20s: HashMap = HashMap::new(); @@ -250,6 +312,7 @@ pub fn link_module<'a>( libcall_trampolines, trampoline_len, &mut riscv_pcrel_hi20s, + &got_info, ); } } @@ -264,6 +327,7 @@ pub fn link_module<'a>( libcall_trampolines, trampoline_len, &mut riscv_pcrel_hi20s, + &got_info, ); } } diff --git a/lib/compiler/src/engine/unwind/systemv.rs b/lib/compiler/src/engine/unwind/systemv.rs index 545c71aa6a8..57a2e60346f 100644 --- a/lib/compiler/src/engine/unwind/systemv.rs +++ b/lib/compiler/src/engine/unwind/systemv.rs @@ -19,6 +19,103 @@ extern "C" { fn __deregister_frame(fde: *const u8); } +// Apple-specific unwind functions - the following is taken from LLVM's libunwind itself. +#[cfg(all(target_os = "macos", target_arch = "aarch64"))] +mod _apple_uw { + use std::{ + collections::HashMap, + sync::{LazyLock, Mutex}, + }; + + static ADDRESSES_MAP: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::default())); + + #[repr(C)] + /// Holds a description of the object-format-header (if any) and unwind info + /// sections for a given address: + /// + /// * dso_base should point to a header for the JIT'd object containing the + /// given address. The header's type should match the format type that + /// libunwind was compiled for (so a mach_header or mach_header_64 on Darwin). + /// A value of zero indicates that no such header exists. + /// + /// * dwarf_section and dwarf_section_length hold the address range of a DWARF + /// eh-frame section associated with the given address, if any. If the + /// dwarf_section_length field is zero it indicates that no such section + /// exists (and in this case dwarf_section should also be set to zero). + /// + /// * compact_unwind_section and compact_unwind_section_length hold the address + /// range of a compact-unwind info section associated with the given address, + /// if any. If the compact_unwind_section_length field is zero it indicates + /// that no such section exists (and in this case compact_unwind_section + /// should also be set to zero). + #[derive(Debug)] + pub struct UnwDynamicUnwindSections { + dso_base: usize, + dwarf_section: usize, + dwarf_section_length: usize, + compact_unwind_section: usize, + compact_unwind_section_length: usize, + } + + // Typedef for unwind-info lookup callbacks. Functions of this type can be + // registered and deregistered using __unw_add_find_dynamic_unwind_sections + // and __unw_remove_find_dynamic_unwind_sections respectively. + // + // An unwind-info lookup callback should return 1 to indicate that it found + // unwind-info for the given address, or 0 to indicate that it did not find + // unwind-info for the given address. If found, the callback should populate + // some or all of the fields of the info argument (which is guaranteed to be + // non-null with all fields zero-initialized): + type UnwFindDynamicUnwindSections = + unsafe extern "C" fn(addr: usize, info: *mut UnwDynamicUnwindSections) -> u32; + + unsafe extern "C" fn x(_addr: usize, _info: *mut UnwDynamicUnwindSections) -> u32 { + todo!() + } + + pub unsafe fn generate_find_dynamic_unwind_sections( + ptr: usize, + len: usize, + ) -> UnwFindDynamicUnwindSections { + let bytes = std::slice::from_raw_parts(std::mem::transmute::(ptr), len); + let p = macho_unwind_info::UnwindInfo::parse(bytes).unwrap(); + let mut funcs = p.functions(); + + while let Ok(Some(f)) = funcs.next() { + println!("{f:?}"); + } + + x + } + + extern "C" { + // Register a dynamic unwind-info lookup callback. If libunwind does not find + // unwind info for a given frame in the executable program or normal dynamic + // shared objects then it will call all registered dynamic lookup functions + // in registration order until either one of them returns true, or the end + // of the list is reached. This lookup will happen before libunwind searches + // any eh-frames registered via __register_frame or + // __unw_add_dynamic_eh_frame_section. + // + // Returns UNW_ESUCCESS for successful registrations. If the given callback + // has already been registered then UNW_EINVAL will be returned. If all + // available callback entries are in use then UNW_ENOMEM will be returned. + pub fn __unw_add_find_dynamic_unwind_sections( + find_dynamic_unwind_sections: UnwFindDynamicUnwindSections, + ) -> u32; + + // Deregister a dynacim unwind-info lookup callback. + // + // Returns UNW_ESUCCESS for successful deregistrations. If the given callback + // has already been registered then UNW_EINVAL will be returned. + pub fn __unw_remove_find_dynamic_unwind_sections( + find_dynamic_unwind_sections: &UnwDynamicUnwindSections, + ) -> u32; + + } +} + /// There are two primary unwinders on Unix platforms: libunwind and libgcc. /// /// Unfortunately their interface to `__register_frame` is different. The @@ -151,6 +248,7 @@ impl UnwindRegistry { // Skip over the CIE and zero-length FDEs. // LLVM's libunwind emits a warning on zero-length FDEs. if current != start && len != 0 { + println!("Hehe!"); __register_frame(current); self.registrations.push(current as usize); } @@ -160,6 +258,26 @@ impl UnwindRegistry { } } } + + pub(crate) fn add_compact_unwind( + &self, + compact_unwind: Option<(wasmer_vm::SectionBodyPtr, usize)>, + ) -> Result<(), String> { + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + unsafe { + if let Some((ptr, len)) = compact_unwind { + _apple_uw::__unw_add_find_dynamic_unwind_sections( + _apple_uw::generate_find_dynamic_unwind_sections((*ptr) as usize, len), + ); + } + } + + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + { + _ = compact_unwind; + } + Ok(()) + } } impl Drop for UnwindRegistry { diff --git a/lib/compiler/src/object/module.rs b/lib/compiler/src/object/module.rs index c93330cb57d..02321968e03 100644 --- a/lib/compiler/src/object/module.rs +++ b/lib/compiler/src/object/module.rs @@ -165,7 +165,7 @@ pub fn emit_compilation( .custom_sections .into_iter() .map(|(section_index, custom_section)| { - if debug_index.map_or(false, |d| d == section_index) { + if debug_index.map_or(false, |d| d == Some(section_index)) { // If this is the debug section let segment = obj.segment_name(StandardSegment::Debug).to_vec(); let section_id = @@ -279,7 +279,7 @@ pub fn emit_compilation( } for (section_index, relocations) in custom_section_relocations.into_iter() { - if !debug_index.map_or(false, |d| d == section_index) { + if !debug_index.map_or(false, |d| d == Some(section_index)) { // Skip DWARF relocations just yet let (section_id, symbol_id) = custom_section_ids.get(section_index).unwrap(); all_relocations.push((*section_id, *symbol_id, relocations)); diff --git a/lib/compiler/src/serialize.rs b/lib/compiler/src/serialize.rs index a05a272b10e..5d01a20031d 100644 --- a/lib/compiler/src/serialize.rs +++ b/lib/compiler/src/serialize.rs @@ -5,7 +5,7 @@ #![allow(missing_docs)] use crate::types::{ - function::{CompiledFunctionFrameInfo, Dwarf, FunctionBody}, + function::{CompiledFunctionFrameInfo, Dwarf, FunctionBody, GOT}, module::CompileModuleInfo, relocation::Relocation, section::{CustomSection, SectionIndex}, @@ -36,6 +36,7 @@ pub struct SerializableCompilation { pub custom_section_relocations: PrimaryMap>, // The section indices corresponding to the Dwarf debug info pub debug: Option, + pub got: Option, // Custom section containing libcall trampolines. pub libcall_trampolines: SectionIndex, // Length of each libcall trampoline. diff --git a/lib/compiler/src/types/function.rs b/lib/compiler/src/types/function.rs index 4b4f8d45e1f..6fb89bed447 100644 --- a/lib/compiler/src/types/function.rs +++ b/lib/compiler/src/types/function.rs @@ -9,9 +9,11 @@ //! A `Compilation` contains the compiled function bodies for a WebAssembly //! module (`CompiledFunction`). +use std::collections::HashMap; + use super::{ address_map::FunctionAddressMap, - relocation::Relocation, + relocation::{Relocation, RelocationTarget}, section::{CustomSection, SectionIndex}, unwind::{ ArchivedCompiledFunctionUnwindInfo, CompiledFunctionUnwindInfo, @@ -120,7 +122,7 @@ pub type Functions = PrimaryMap; /// The custom sections for a Compilation. pub type CustomSections = PrimaryMap; -/// The DWARF information for this Compilation. +/// The unwinding information for this Compilation. /// /// It is used for retrieving the unwind information once an exception /// happens. @@ -134,13 +136,43 @@ pub struct Dwarf { /// The section index in the [`Compilation`] that corresponds to the exception frames. /// [Learn /// more](https://refspecs.linuxfoundation.org/LSB_3.0.0/LSB-PDA/LSB-PDA/ehframechpt.html). - pub eh_frame: SectionIndex, + pub eh_frame: Option, + pub compact_unwind: Option, +} + +/// The GOT - Global Offset Table - for this Compilation. +/// +/// The GOT is but a list of pointers to objects (functions, data, sections..); in our context the +/// GOT is represented simply as a custom section. +/// +/// This data structure holds the index of the related custom section and a map between +/// [`RelocationTarget`] and the entry number in the GOT; that is, for a relocation target `r` one +/// can find its address in the got as `r_addr = custom_sections[GOT_index][GOT_map[r]]`. +#[cfg_attr(feature = "enable-serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "artifact-size", derive(loupe::MemoryUsage))] +#[derive(RkyvSerialize, RkyvDeserialize, Archive, Debug, PartialEq, Eq, Clone)] +#[rkyv(derive(Debug))] +pub struct GOT { + /// The section index in the [`Compilation`] that corresponds to the GOT. + pub index: SectionIndex, + /// The map between relocation targets and their index in the GOT. + pub map: HashMap, } impl Dwarf { /// Creates a `Dwarf` struct with the corresponding indices for its sections pub fn new(eh_frame: SectionIndex) -> Self { - Self { eh_frame } + Self { + eh_frame: Some(eh_frame), + compact_unwind: None, + } + } + + pub fn new_cu(compact_unwind: SectionIndex) -> Self { + Self { + eh_frame: None, + compact_unwind: Some(compact_unwind), + } } } @@ -190,4 +222,7 @@ pub struct Compilation { /// Section ids corresponding to the Dwarf debug info pub debug: Option, + + /// An optional reference to the [`GOT`]. + pub got: Option, } diff --git a/lib/compiler/src/types/relocation.rs b/lib/compiler/src/types/relocation.rs index 3ab78d959ca..295a3674503 100644 --- a/lib/compiler/src/types/relocation.rs +++ b/lib/compiler/src/types/relocation.rs @@ -89,6 +89,67 @@ pub enum RelocationKind { ElfX86_64TlsGd, // /// Mach-O x86_64 32 bit signed PC relative offset to a `__thread_vars` entry. // MachOX86_64Tlv, + + // -- Mach-O-specific relocations + // + // --- Arm64 + // (MACHO_ARM64_RELOC_UNSIGNED) for pointers + MachoArm64RelocUnsigned, + // (MACHO_ARM64_RELOC_SUBTRACTOR) must be followed by a ARM64_RELOC_UNSIGNED + MachoArm64RelocSubtractor, + // (MACHO_ARM64_RELOC_BRANCH26) a B/BL instruction with 26-bit displacement + MachoArm64RelocBranch26, + // (MACHO_ARM64_RELOC_PAGE21) pc-rel distance to page of target + MachoArm64RelocPage21, + // (MACHO_ARM64_RELOC_PAGEOFF12) offset within page, scaled by r_length + MachoArm64RelocPageoff12, + // (MACHO_ARM64_RELOC_GOT_LOAD_PAGE21) pc-rel distance to page of GOT slot + MachoArm64RelocGotLoadPage21, + // (MACHO_ARM64_RELOC_GOT_LOAD_PAGEOFF12) offset within page of GOT slot, scaled by r_length + MachoArm64RelocGotLoadPageoff12, + // (MACHO_ARM64_RELOC_POINTER_TO_GOT) for pointers to GOT slots + MachoArm64RelocPointerToGot, + // (MACHO_ARM64_RELOC_TLVP_LOAD_PAGE21) pc-rel distance to page of TLVP slot + MachoArm64RelocTlvpLoadPage21, + // (MACHO_ARM64_RELOC_TLVP_LOAD_PAGEOFF12) offset within page of TLVP slot, scaled by r_length + MachoArm64RelocTlvpLoadPageoff12, + // (MACHO_ARM64_RELOC_ADDEND) must be followed by PAGE21 or PAGEOFF12 + MachoArm64RelocAddend, + + // --- X86_64 + // (MACHO_X86_64_RELOC_UNSIGNED) for absolute addresses + MachoX86_64RelocUnsigned, + // (MACHO_X86_64_RELOC_SIGNED) for signed 32-bit displacement + MachoX86_64RelocSigned, + // (MACHO_X86_64_RELOC_BRANCH) a CALL/JMP instruction with 32-bit displacement + MachoX86_64RelocBranch, + // (MACHO_X86_64_RELOC_GOT_LOAD) a MOVQ load of a GOT entry + MachoX86_64RelocGotLoad, + // (MACHO_X86_64_RELOC_GOT) other GOT references + MachoX86_64RelocGot, + // (MACHO_X86_64_RELOC_SUBTRACTOR) must be followed by a X86_64_RELOC_UNSIGNED + MachoX86_64RelocSubtractor, + // (MACHO_X86_64_RELOC_SIGNED_1) for signed 32-bit displacement with a -1 addend + MachoX86_64RelocSigned1, + // (MACHO_X86_64_RELOC_SIGNED_2) for signed 32-bit displacement with a -2 addend + MachoX86_64RelocSigned2, + // (MACHO_X86_64_RELOC_SIGNED_4) for signed 32-bit displacement with a -4 addend + MachoX86_64RelocSigned4, + // (MACHO_X86_64_RELOC_TLV) for thread local variables + MachoX86_64RelocTlv, +} + +impl RelocationKind { + pub fn needs_got(&self) -> bool { + match self { + RelocationKind::MachoArm64RelocGotLoadPage21 + | RelocationKind::MachoArm64RelocGotLoadPageoff12 + | RelocationKind::MachoArm64RelocPointerToGot + | RelocationKind::MachoX86_64RelocGotLoad + | RelocationKind::MachoX86_64RelocGot => true, + _ => false, + } + } } impl fmt::Display for RelocationKind { @@ -120,7 +181,27 @@ impl fmt::Display for RelocationKind { Self::Aarch64AddAbsLo12Nc => write!(f, "Aarch64AddAbsLo12Nc"), Self::Aarch64Ldst128AbsLo12Nc => write!(f, "Aarch64Ldst128AbsLo12Nc"), Self::Aarch64Ldst64AbsLo12Nc => write!(f, "Aarch64Ldst64AbsLo12Nc"), - // Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"), + Self::MachoArm64RelocUnsigned => write!(f, "MachoArm64RelocUnsigned"), + Self::MachoArm64RelocSubtractor => write!(f, "MachoArm64RelocSubtractor"), + Self::MachoArm64RelocBranch26 => write!(f, "MachoArm64RelocBranch26"), + Self::MachoArm64RelocPage21 => write!(f, "MachoArm64RelocPage21"), + Self::MachoArm64RelocPageoff12 => write!(f, "MachoArm64RelocPageoff12"), + Self::MachoArm64RelocGotLoadPage21 => write!(f, "MachoArm64RelocGotLoadPage21"), + Self::MachoArm64RelocGotLoadPageoff12 => write!(f, "MachoArm64RelocGotLoadPageoff12"), + Self::MachoArm64RelocPointerToGot => write!(f, "MachoArm64RelocPointerToGot"), + Self::MachoArm64RelocTlvpLoadPage21 => write!(f, "MachoArm64RelocTlvpLoadPage21"), + Self::MachoArm64RelocTlvpLoadPageoff12 => write!(f, "MachoArm64RelocTlvpLoadPageoff12"), + Self::MachoArm64RelocAddend => write!(f, "MachoArm64RelocAddend"), + Self::MachoX86_64RelocUnsigned => write!(f, "MachoX86_64RelocUnsigned"), + Self::MachoX86_64RelocSigned => write!(f, "MachoX86_64RelocSigned"), + Self::MachoX86_64RelocBranch => write!(f, "MachoX86_64RelocBranch"), + Self::MachoX86_64RelocGotLoad => write!(f, "MachoX86_64RelocGotLoad"), + Self::MachoX86_64RelocGot => write!(f, "MachoX86_64RelocGot"), + Self::MachoX86_64RelocSubtractor => write!(f, "MachoX86_64RelocSubtractor"), + Self::MachoX86_64RelocSigned1 => write!(f, "MachoX86_64RelocSigned1"), + Self::MachoX86_64RelocSigned2 => write!(f, "MachoX86_64RelocSigned2"), + Self::MachoX86_64RelocSigned4 => write!(f, "MachoX86_64RelocSigned4"), + Self::MachoX86_64RelocTlv => write!(f, "MachoX86_64RelocTlv"), } } } @@ -172,7 +253,11 @@ pub trait RelocationLike { | RelocationKind::Arm64Movw3 | RelocationKind::RiscvPCRelLo12I | RelocationKind::Aarch64Ldst128AbsLo12Nc - | RelocationKind::Aarch64Ldst64AbsLo12Nc => { + | RelocationKind::Aarch64Ldst64AbsLo12Nc + | RelocationKind::MachoArm64RelocUnsigned + | RelocationKind::MachoX86_64RelocUnsigned + | RelocationKind::MachoArm64RelocSubtractor + | RelocationKind::MachoX86_64RelocSubtractor => { let reloc_address = start + self.offset() as usize; let reloc_addend = self.addend() as isize; let reloc_abs = target_func_address @@ -231,7 +316,7 @@ pub trait RelocationLike { .wrapping_add(reloc_addend as u64); (reloc_address, reloc_delta_u32) } - RelocationKind::Aarch64AdrPrelPgHi21 => { + RelocationKind::Aarch64AdrPrelPgHi21 | RelocationKind::MachoArm64RelocGotLoadPage21 => { let reloc_address = start + self.offset() as usize; let reloc_addend = self.addend() as isize; let target_page = @@ -239,6 +324,18 @@ pub trait RelocationLike { let pc_page = reloc_address & !(0xFFF); (reloc_address, target_page.wrapping_sub(pc_page) as u64) } + RelocationKind::MachoArm64RelocGotLoadPageoff12 + | RelocationKind::MachoArm64RelocPointerToGot => { + let reloc_address = start + self.offset() as usize; + let reloc_addend = self.addend() as isize; + let target_offset = + (target_func_address.wrapping_add(reloc_addend as u64) & (0xFFF)) as usize; + (reloc_address, target_offset as u64) + } + //RelocationKind::MachoArm64RelocPointerToGot => { + // let reloc_address = start + self.offset() as usize; + // (reloc_address, target_func_address) + //} _ => panic!("Relocation kind unsupported"), } } @@ -283,8 +380,8 @@ impl RelocationLike for ArchivedRelocation { /// Destination function. Can be either user function or some special one, like `memory.grow`. #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "artifact-size", derive(loupe::MemoryUsage))] -#[derive(RkyvSerialize, RkyvDeserialize, Archive, Debug, Copy, Clone, PartialEq, Eq)] -#[rkyv(derive(Debug), compare(PartialEq))] +#[derive(RkyvSerialize, RkyvDeserialize, Archive, Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[rkyv(derive(Debug, Hash, PartialEq, Eq), compare(PartialEq))] #[repr(u8)] pub enum RelocationTarget { /// A relocation to a function defined locally in the wasm (not an imported one). diff --git a/lib/compiler/src/types/section.rs b/lib/compiler/src/types/section.rs index 00ced395472..eb144ab4596 100644 --- a/lib/compiler/src/types/section.rs +++ b/lib/compiler/src/types/section.rs @@ -35,7 +35,7 @@ use wasmer_types::entity_impl; )] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "artifact-size", derive(loupe::MemoryUsage))] -#[rkyv(derive(Debug), compare(PartialEq, PartialOrd))] +#[rkyv(derive(Debug, Hash, PartialEq, Eq), compare(PartialEq, PartialOrd))] pub struct SectionIndex(u32); entity_impl!(SectionIndex); diff --git a/lib/compiler/src/types/unwind.rs b/lib/compiler/src/types/unwind.rs index ce2ad70d77e..819a86ad7e4 100644 --- a/lib/compiler/src/types/unwind.rs +++ b/lib/compiler/src/types/unwind.rs @@ -35,6 +35,7 @@ pub enum CompiledFunctionUnwindInfo { pub enum CompiledFunctionUnwindInfoReference<'a> { WindowsX64(&'a [u8]), Dwarf, + CompactUnwind, } /// Any struct that acts like a `CompiledFunctionUnwindInfo`.