diff --git a/src/jit.rs b/src/jit.rs index 668c6a18..8fccdde1 100644 --- a/src/jit.rs +++ b/src/jit.rs @@ -158,9 +158,11 @@ impl JitProgram { } // Special values for target_pc in struct Jump -const TARGET_PC_TRACE: usize = std::usize::MAX - 13; -const TARGET_PC_TRANSLATE_PC: usize = std::usize::MAX - 12; -const TARGET_PC_TRANSLATE_PC_LOOP: usize = std::usize::MAX - 11; +const TARGET_PC_TRACE: usize = std::usize::MAX - 29; +const TARGET_PC_TRANSLATE_PC: usize = std::usize::MAX - 28; +const TARGET_PC_TRANSLATE_PC_LOOP: usize = std::usize::MAX - 27; +const TARGET_PC_TRANSLATE_MEMORY_ADDRESS: usize = std::usize::MAX - 26; +const TARGET_PC_MEMORY_ACCESS_VIOLATION: usize = std::usize::MAX - 18; const TARGET_PC_CALL_EXCEEDED_MAX_INSTRUCTIONS: usize = std::usize::MAX - 10; const TARGET_PC_CALL_DEPTH_EXCEEDED: usize = std::usize::MAX - 9; const TARGET_PC_CALL_OUTSIDE_TEXT_SEGMENT: usize = std::usize::MAX - 8; @@ -707,21 +709,29 @@ fn emit_rust_call(jit: &mut JitCompiler, function: *const u #[inline] fn emit_address_translation(jit: &mut JitCompiler, host_addr: u8, vm_addr: Value, len: u64, access_type: AccessType) -> Result<(), EbpfError> { - emit_rust_call(jit, MemoryMapping::map:: as *const u8, &[ - Argument { index: 3, value: vm_addr }, // Specify first as the src register could be overwritten by other arguments - Argument { index: 4, value: Value::Constant64(len as i64, false) }, - Argument { index: 2, value: Value::Constant64(access_type as i64, false) }, - Argument { index: 1, value: Value::RegisterPlusConstant32(R10, jit.program_argument_key, false) }, // JitProgramArgument::memory_mapping - Argument { index: 0, value: Value::RegisterIndirect(RBP, slot_on_environment_stack(jit, EnvironmentStackSlot::OptRetValPtr), false) }, - ], None, true)?; - - // Throw error if the result indicates one - X86Instruction::load_immediate(OperandSize::S64, R11, jit.pc as i64).emit(jit)?; - emit_jcc(jit, 0x85, TARGET_PC_EXCEPTION_AT)?; - - // Store Ok value in result register - X86Instruction::load(OperandSize::S64, RBP, R11, X86IndirectAccess::Offset(slot_on_environment_stack(jit, EnvironmentStackSlot::OptRetValPtr))).emit(jit)?; - X86Instruction::load(OperandSize::S64, R11, host_addr, X86IndirectAccess::Offset(8)).emit(jit) + match vm_addr { + Value::RegisterPlusConstant64(reg, constant, user_provided) => { + if user_provided && jit.config.sanitize_user_provided_values { + emit_sanitized_load_immediate(jit, OperandSize::S64, R11, constant)?; + } else { + X86Instruction::load_immediate(OperandSize::S64, R11, constant).emit(jit)?; + } + emit_alu(jit, OperandSize::S64, 0x01, reg, R11, 0, None)?; + }, + Value::Constant64(constant, user_provided) => { + if user_provided && jit.config.sanitize_user_provided_values { + emit_sanitized_load_immediate(jit, OperandSize::S64, R11, constant)?; + } else { + X86Instruction::load_immediate(OperandSize::S64, R11, constant).emit(jit)?; + } + }, + _ => { + #[cfg(debug_assertions)] + unreachable!(); + }, + } + emit_call(jit, TARGET_PC_TRANSLATE_MEMORY_ADDRESS + len.trailing_zeros() as usize + 4 * (access_type as usize))?; + X86Instruction::mov(OperandSize::S64, R11, host_addr).emit(jit) } fn emit_shift(jit: &mut JitCompiler, size: OperandSize, opcode_extension: u8, source: u8, destination: u8, immediate: Option) -> Result<(), EbpfError> { @@ -1354,7 +1364,56 @@ impl JitCompiler { emit_alu(self, OperandSize::S64, 0x29, REGISTER_MAP[0], R11, 0, None)?; // R11 -= REGISTER_MAP[0]; emit_alu(self, OperandSize::S64, 0xc1, 5, R11, 3, None)?; // R11 >>= 3; X86Instruction::pop(REGISTER_MAP[0]).emit(self)?; // Restore REGISTER_MAP[0] - X86Instruction::return_near().emit(self) + X86Instruction::return_near().emit(self)?; + + // Translates a vm memory address to a host memory address + for (access_type, len) in &[ + (AccessType::Load, 1i64), + (AccessType::Load, 2i64), + (AccessType::Load, 4i64), + (AccessType::Load, 8i64), + (AccessType::Store, 1i64), + (AccessType::Store, 2i64), + (AccessType::Store, 4i64), + (AccessType::Store, 8i64), + ] { + let target_offset = len.trailing_zeros() as usize + 4 * (*access_type as usize); + + set_anchor(self, TARGET_PC_TRANSLATE_MEMORY_ADDRESS + target_offset); + X86Instruction::push(R11).emit(self)?; + emit_rust_call(self, MemoryMapping::map:: as *const u8, &[ + Argument { index: 3, value: Value::Register(R11) }, // Specify first as the src register could be overwritten by other arguments + Argument { index: 4, value: Value::Constant64(*len, false) }, + Argument { index: 2, value: Value::Constant64(*access_type as i64, false) }, + Argument { index: 1, value: Value::RegisterPlusConstant32(R10, self.program_argument_key, false) }, // JitProgramArgument::memory_mapping + Argument { index: 0, value: Value::RegisterIndirect(RBP, slot_on_environment_stack(self, EnvironmentStackSlot::OptRetValPtr), false) }, // Pointer to optional typed return value + ], None, true)?; + + // Throw error if the result indicates one + emit_jcc(self, 0x85, TARGET_PC_MEMORY_ACCESS_VIOLATION + target_offset)?; + + // Store Ok value in result register + X86Instruction::load(OperandSize::S64, RBP, R11, X86IndirectAccess::Offset(slot_on_environment_stack(self, EnvironmentStackSlot::OptRetValPtr))).emit(self)?; + X86Instruction::load(OperandSize::S64, R11, R11, X86IndirectAccess::Offset(8)).emit(self)?; + emit_alu(self, OperandSize::S64, 0x81, 0, RSP, 8, None)?; + X86Instruction::return_near().emit(self)?; + + set_anchor(self, TARGET_PC_MEMORY_ACCESS_VIOLATION + target_offset); + emit_alu(self, OperandSize::S64, 0x31, R11, R11, 0, None)?; // R11 = 0; + X86Instruction::load(OperandSize::S64, RSP, R11, X86IndirectAccess::OffsetIndexShift(0, R11, 0)).emit(self)?; + emit_rust_call(self, MemoryMapping::generate_access_violation:: as *const u8, &[ + Argument { index: 3, value: Value::Register(R11) }, // Specify first as the src register could be overwritten by other arguments + Argument { index: 4, value: Value::Constant64(*len, false) }, + Argument { index: 2, value: Value::Constant64(*access_type as i64, false) }, + Argument { index: 1, value: Value::RegisterPlusConstant32(R10, self.program_argument_key, false) }, // JitProgramArgument::memory_mapping + Argument { index: 0, value: Value::RegisterIndirect(RBP, slot_on_environment_stack(self, EnvironmentStackSlot::OptRetValPtr), false) }, // Pointer to optional typed return value + ], None, true)?; + X86Instruction::pop(R11).emit(self)?; + X86Instruction::pop(R11).emit(self)?; + emit_call(self, TARGET_PC_TRANSLATE_PC)?; + emit_jmp(self, TARGET_PC_EXCEPTION_AT)?; + } + Ok(()) } fn generate_exception_handlers(&mut self) -> Result<(), EbpfError> { diff --git a/src/memory_region.rs b/src/memory_region.rs index 3e3b575b..ea3a9b8b 100644 --- a/src/memory_region.rs +++ b/src/memory_region.rs @@ -85,7 +85,7 @@ impl std::cmp::Ord for MemoryRegion { } /// Type of memory access -#[derive(Clone, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum AccessType { /// Read Load, @@ -153,7 +153,7 @@ impl<'a> MemoryMapping<'a> { } index >>= index.trailing_zeros() + 1; if index == 0 { - return Err(self.generate_access_violation(access_type, vm_addr, len)); + return self.generate_access_violation(access_type, vm_addr, len); } let region = &self.regions[index - 1]; if access_type == AccessType::Load || region.is_writable { @@ -161,43 +161,26 @@ impl<'a> MemoryMapping<'a> { return Ok(host_addr); } } - Err(self.generate_access_violation(access_type, vm_addr, len)) - } - - /// Resize the memory_region at the given index - pub fn resize_region( - &mut self, - index: usize, - new_len: u64, - ) -> Result<(), EbpfError> { - if index < self.regions.len() - 1 - && self.regions[index].vm_addr.saturating_add(new_len) > self.regions[index + 1].vm_addr - { - return Err(EbpfError::VirtualAddressOverlap( - self.regions[index + 1].vm_addr, - )); - } - self.regions[index].len = new_len; - Ok(()) + self.generate_access_violation(access_type, vm_addr, len) } /// Helper for map to generate errors - fn generate_access_violation( + pub fn generate_access_violation( &self, access_type: AccessType, vm_addr: u64, len: u64, - ) -> EbpfError { + ) -> Result> { let stack_frame = (vm_addr as i64 - ebpf::MM_STACK_START as i64) / self.config.stack_frame_size as i64; if (-1..self.config.max_call_depth as i64 + 1).contains(&stack_frame) { - EbpfError::StackAccessViolation( + Err(EbpfError::StackAccessViolation( 0, // Filled out later access_type, vm_addr, len, stack_frame, - ) + )) } else { let region_name = match vm_addr & !(ebpf::MM_PROGRAM_START - 1) { ebpf::MM_PROGRAM_START => "program", @@ -206,13 +189,30 @@ impl<'a> MemoryMapping<'a> { ebpf::MM_INPUT_START => "input", _ => "unknown", }; - EbpfError::AccessViolation( + Err(EbpfError::AccessViolation( 0, // Filled out later access_type, vm_addr, len, region_name, - ) + )) } } + + /// Resize the memory_region at the given index + pub fn resize_region( + &mut self, + index: usize, + new_len: u64, + ) -> Result<(), EbpfError> { + if index < self.regions.len() - 1 + && self.regions[index].vm_addr.saturating_add(new_len) > self.regions[index + 1].vm_addr + { + return Err(EbpfError::VirtualAddressOverlap( + self.regions[index + 1].vm_addr, + )); + } + self.regions[index].len = new_len; + Ok(()) + } }