Skip to content
This repository has been archived by the owner on Jan 10, 2025. It is now read-only.

Stop Inlining Address Translation #192

Merged
merged 2 commits into from
Jul 12, 2021
Merged

Stop Inlining Address Translation #192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
973d8e8
Moves address translation in JIT to a helper routine.
Lichtso Jul 9, 2021
0efb7d7
Switches to direct usage of MemoryMapping::generate_access_violation(…
Lichtso Jul 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 78 additions & 19 deletions src/jit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,11 @@ impl<E: UserDefinedError, I: InstructionMeter> JitProgram<E, I> {
}

// Special values for target_pc in struct Jump
const TARGET_PC_TRACE: usize = std::usize::MAX - 13;
const TARGET_PC_TRANSLATE_PC: usize = std::usize::MAX - 12;
const TARGET_PC_TRANSLATE_PC_LOOP: usize = std::usize::MAX - 11;
const TARGET_PC_TRACE: usize = std::usize::MAX - 29;
const TARGET_PC_TRANSLATE_PC: usize = std::usize::MAX - 28;
const TARGET_PC_TRANSLATE_PC_LOOP: usize = std::usize::MAX - 27;
const TARGET_PC_TRANSLATE_MEMORY_ADDRESS: usize = std::usize::MAX - 26;
const TARGET_PC_MEMORY_ACCESS_VIOLATION: usize = std::usize::MAX - 18;
const TARGET_PC_CALL_EXCEEDED_MAX_INSTRUCTIONS: usize = std::usize::MAX - 10;
const TARGET_PC_CALL_DEPTH_EXCEEDED: usize = std::usize::MAX - 9;
const TARGET_PC_CALL_OUTSIDE_TEXT_SEGMENT: usize = std::usize::MAX - 8;
Expand Down Expand Up @@ -707,21 +709,29 @@ fn emit_rust_call<E: UserDefinedError>(jit: &mut JitCompiler, function: *const u

#[inline]
fn emit_address_translation<E: UserDefinedError>(jit: &mut JitCompiler, host_addr: u8, vm_addr: Value, len: u64, access_type: AccessType) -> Result<(), EbpfError<E>> {
emit_rust_call(jit, MemoryMapping::map::<UserError> as *const u8, &[
Argument { index: 3, value: vm_addr }, // Specify first as the src register could be overwritten by other arguments
Argument { index: 4, value: Value::Constant64(len as i64, false) },
Argument { index: 2, value: Value::Constant64(access_type as i64, false) },
Argument { index: 1, value: Value::RegisterPlusConstant32(R10, jit.program_argument_key, false) }, // JitProgramArgument::memory_mapping
Argument { index: 0, value: Value::RegisterIndirect(RBP, slot_on_environment_stack(jit, EnvironmentStackSlot::OptRetValPtr), false) },
], None, true)?;

// Throw error if the result indicates one
X86Instruction::load_immediate(OperandSize::S64, R11, jit.pc as i64).emit(jit)?;
emit_jcc(jit, 0x85, TARGET_PC_EXCEPTION_AT)?;

// Store Ok value in result register
X86Instruction::load(OperandSize::S64, RBP, R11, X86IndirectAccess::Offset(slot_on_environment_stack(jit, EnvironmentStackSlot::OptRetValPtr))).emit(jit)?;
X86Instruction::load(OperandSize::S64, R11, host_addr, X86IndirectAccess::Offset(8)).emit(jit)
match vm_addr {
Value::RegisterPlusConstant64(reg, constant, user_provided) => {
if user_provided && jit.config.sanitize_user_provided_values {
emit_sanitized_load_immediate(jit, OperandSize::S64, R11, constant)?;
} else {
X86Instruction::load_immediate(OperandSize::S64, R11, constant).emit(jit)?;
}
emit_alu(jit, OperandSize::S64, 0x01, reg, R11, 0, None)?;
},
Value::Constant64(constant, user_provided) => {
if user_provided && jit.config.sanitize_user_provided_values {
emit_sanitized_load_immediate(jit, OperandSize::S64, R11, constant)?;
} else {
X86Instruction::load_immediate(OperandSize::S64, R11, constant).emit(jit)?;
}
},
_ => {
#[cfg(debug_assertions)]
unreachable!();
},
}
emit_call(jit, TARGET_PC_TRANSLATE_MEMORY_ADDRESS + len.trailing_zeros() as usize + 4 * (access_type as usize))?;
X86Instruction::mov(OperandSize::S64, R11, host_addr).emit(jit)
}

fn emit_shift<E: UserDefinedError>(jit: &mut JitCompiler, size: OperandSize, opcode_extension: u8, source: u8, destination: u8, immediate: Option<i64>) -> Result<(), EbpfError<E>> {
Expand Down Expand Up @@ -1354,7 +1364,56 @@ impl JitCompiler {
emit_alu(self, OperandSize::S64, 0x29, REGISTER_MAP[0], R11, 0, None)?; // R11 -= REGISTER_MAP[0];
emit_alu(self, OperandSize::S64, 0xc1, 5, R11, 3, None)?; // R11 >>= 3;
X86Instruction::pop(REGISTER_MAP[0]).emit(self)?; // Restore REGISTER_MAP[0]
X86Instruction::return_near().emit(self)
X86Instruction::return_near().emit(self)?;

// Translates a vm memory address to a host memory address
for (access_type, len) in &[
(AccessType::Load, 1i64),
(AccessType::Load, 2i64),
(AccessType::Load, 4i64),
(AccessType::Load, 8i64),
(AccessType::Store, 1i64),
(AccessType::Store, 2i64),
(AccessType::Store, 4i64),
(AccessType::Store, 8i64),
] {
let target_offset = len.trailing_zeros() as usize + 4 * (*access_type as usize);

set_anchor(self, TARGET_PC_TRANSLATE_MEMORY_ADDRESS + target_offset);
X86Instruction::push(R11).emit(self)?;
emit_rust_call(self, MemoryMapping::map::<UserError> as *const u8, &[
Argument { index: 3, value: Value::Register(R11) }, // Specify first as the src register could be overwritten by other arguments
Argument { index: 4, value: Value::Constant64(*len, false) },
Argument { index: 2, value: Value::Constant64(*access_type as i64, false) },
Argument { index: 1, value: Value::RegisterPlusConstant32(R10, self.program_argument_key, false) }, // JitProgramArgument::memory_mapping
Argument { index: 0, value: Value::RegisterIndirect(RBP, slot_on_environment_stack(self, EnvironmentStackSlot::OptRetValPtr), false) }, // Pointer to optional typed return value
], None, true)?;

// Throw error if the result indicates one
emit_jcc(self, 0x85, TARGET_PC_MEMORY_ACCESS_VIOLATION + target_offset)?;

// Store Ok value in result register
X86Instruction::load(OperandSize::S64, RBP, R11, X86IndirectAccess::Offset(slot_on_environment_stack(self, EnvironmentStackSlot::OptRetValPtr))).emit(self)?;
X86Instruction::load(OperandSize::S64, R11, R11, X86IndirectAccess::Offset(8)).emit(self)?;
emit_alu(self, OperandSize::S64, 0x81, 0, RSP, 8, None)?;
X86Instruction::return_near().emit(self)?;

set_anchor(self, TARGET_PC_MEMORY_ACCESS_VIOLATION + target_offset);
emit_alu(self, OperandSize::S64, 0x31, R11, R11, 0, None)?; // R11 = 0;
X86Instruction::load(OperandSize::S64, RSP, R11, X86IndirectAccess::OffsetIndexShift(0, R11, 0)).emit(self)?;
emit_rust_call(self, MemoryMapping::generate_access_violation::<UserError> as *const u8, &[
Argument { index: 3, value: Value::Register(R11) }, // Specify first as the src register could be overwritten by other arguments
Argument { index: 4, value: Value::Constant64(*len, false) },
Argument { index: 2, value: Value::Constant64(*access_type as i64, false) },
Argument { index: 1, value: Value::RegisterPlusConstant32(R10, self.program_argument_key, false) }, // JitProgramArgument::memory_mapping
Argument { index: 0, value: Value::RegisterIndirect(RBP, slot_on_environment_stack(self, EnvironmentStackSlot::OptRetValPtr), false) }, // Pointer to optional typed return value
], None, true)?;
X86Instruction::pop(R11).emit(self)?;
X86Instruction::pop(R11).emit(self)?;
emit_call(self, TARGET_PC_TRANSLATE_PC)?;
emit_jmp(self, TARGET_PC_EXCEPTION_AT)?;
}
Ok(())
}

fn generate_exception_handlers<E: UserDefinedError>(&mut self) -> Result<(), EbpfError<E>> {
Expand Down
52 changes: 26 additions & 26 deletions src/memory_region.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ impl std::cmp::Ord for MemoryRegion {
}

/// Type of memory access
#[derive(Clone, PartialEq, Eq, Debug)]
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum AccessType {
/// Read
Load,
Expand Down Expand Up @@ -153,51 +153,34 @@ impl<'a> MemoryMapping<'a> {
}
index >>= index.trailing_zeros() + 1;
if index == 0 {
return Err(self.generate_access_violation(access_type, vm_addr, len));
return self.generate_access_violation(access_type, vm_addr, len);
}
let region = &self.regions[index - 1];
if access_type == AccessType::Load || region.is_writable {
if let Ok(host_addr) = region.vm_to_host::<E>(vm_addr, len as u64) {
return Ok(host_addr);
}
}
Err(self.generate_access_violation(access_type, vm_addr, len))
}

/// Resize the memory_region at the given index
pub fn resize_region<E: UserDefinedError>(
&mut self,
index: usize,
new_len: u64,
) -> Result<(), EbpfError<E>> {
if index < self.regions.len() - 1
&& self.regions[index].vm_addr.saturating_add(new_len) > self.regions[index + 1].vm_addr
{
return Err(EbpfError::VirtualAddressOverlap(
self.regions[index + 1].vm_addr,
));
}
self.regions[index].len = new_len;
Ok(())
self.generate_access_violation(access_type, vm_addr, len)
}

/// Helper for map to generate errors
fn generate_access_violation<E: UserDefinedError>(
pub fn generate_access_violation<E: UserDefinedError>(
&self,
access_type: AccessType,
vm_addr: u64,
len: u64,
) -> EbpfError<E> {
) -> Result<u64, EbpfError<E>> {
let stack_frame =
(vm_addr as i64 - ebpf::MM_STACK_START as i64) / self.config.stack_frame_size as i64;
if (-1..self.config.max_call_depth as i64 + 1).contains(&stack_frame) {
EbpfError::StackAccessViolation(
Err(EbpfError::StackAccessViolation(
0, // Filled out later
access_type,
vm_addr,
len,
stack_frame,
)
))
} else {
let region_name = match vm_addr & !(ebpf::MM_PROGRAM_START - 1) {
ebpf::MM_PROGRAM_START => "program",
Expand All @@ -206,13 +189,30 @@ impl<'a> MemoryMapping<'a> {
ebpf::MM_INPUT_START => "input",
_ => "unknown",
};
EbpfError::AccessViolation(
Err(EbpfError::AccessViolation(
0, // Filled out later
access_type,
vm_addr,
len,
region_name,
)
))
}
}

/// Resize the memory_region at the given index
pub fn resize_region<E: UserDefinedError>(
&mut self,
index: usize,
new_len: u64,
) -> Result<(), EbpfError<E>> {
if index < self.regions.len() - 1
&& self.regions[index].vm_addr.saturating_add(new_len) > self.regions[index + 1].vm_addr
{
return Err(EbpfError::VirtualAddressOverlap(
self.regions[index + 1].vm_addr,
));
}
self.regions[index].len = new_len;
Ok(())
}
}