diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index b30c4e5cb55e..fb11d31d60bd 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -225,12 +225,12 @@ impl ABIMachineSpec for AArch64MachineDeps { slots: smallvec![ ABIArgSlot::Reg { reg: lower_reg.to_real_reg().unwrap(), - ty: param.value_type, + ty: reg_types[0], extension: param.extension, }, ABIArgSlot::Reg { reg: upper_reg.to_real_reg().unwrap(), - ty: param.value_type, + ty: reg_types[1], extension: param.extension, }, ], diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index db111ba12e47..a21658b8de57 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -2818,3 +2818,11 @@ (let ((dst WritableReg (temp_writable_reg $I8X16)) (_ Unit (emit (MInst.IntToFpu op dst src)))) dst)) + +;;;; Helpers for Emitting Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl gen_call (SigRef ExternalName RelocDistance ValueSlice) InstOutput) +(extern constructor gen_call gen_call) + +(decl gen_call_indirect (SigRef Value ValueSlice) InstOutput) +(extern constructor gen_call_indirect gen_call_indirect) diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 5457039059c1..955f453532e7 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -2016,3 +2016,17 @@ (rule (lower (get_return_address)) (aarch64_link)) + +;;;; Rules for calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (call (func_ref_data sig_ref extname dist) inputs)) + (gen_call sig_ref extname dist inputs)) + +(rule (lower (call_indirect sig_ref val inputs)) + (gen_call_indirect sig_ref val inputs)) + +;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; N.B.: the Ret itself is generated by the ABI. +(rule (lower (return args)) + (lower_return (range 0 (value_slice_len args)) args)) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index 1e190a3fc765..3b876bfbc816 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -2,6 +2,7 @@ // Pull in the ISLE generated code. pub mod generated_code; +use generated_code::Context; // Types that the generated ISLE code uses via `use super::*`. use super::{ @@ -14,6 +15,7 @@ use super::{ use crate::isa::aarch64::inst::{FPULeftShiftImm, FPURightShiftImm}; use crate::isa::aarch64::lower::{lower_address, lower_splat_const}; use crate::isa::aarch64::settings::Flags as IsaFlags; +use crate::machinst::valueregs; use crate::machinst::{isle::*, InputSourceInst}; use crate::settings::Flags; use crate::{ @@ -22,10 +24,11 @@ use crate::{ immediates::*, types::*, AtomicRmwOp, ExternalName, Inst, InstructionData, MemFlags, TrapCode, Value, ValueList, }, + isa::aarch64::abi::{AArch64Caller, AArch64MachineDeps}, isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm}, isa::aarch64::lower::{writable_vreg, writable_xreg, xreg}, isa::unwind::UnwindInst, - machinst::{ty_bits, InsnOutput, Lower, VCodeConstant, VCodeConstantData}, + machinst::{ty_bits, InsnOutput, Lower, MachInst, VCodeConstant, VCodeConstantData}, }; use regalloc2::PReg; use std::boxed::Box; @@ -69,8 +72,13 @@ pub struct SinkableAtomicLoad { atomic_addr: Value, } -impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { +impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { + isle_prelude_method_helpers!(AArch64Caller); +} + +impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { isle_prelude_methods!(); + isle_prelude_caller_methods!(AArch64MachineDeps, AArch64Caller); fn sign_return_address_disabled(&mut self) -> Option<()> { if self.isa_flags.sign_return_address() { diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 10f8f3516df0..30ea4a2a118a 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -5,7 +5,6 @@ use crate::binemit::CodeOffset; use crate::ir::types::*; use crate::ir::Inst as IRInst; use crate::ir::{InstructionData, Opcode}; -use crate::isa::aarch64::abi::*; use crate::isa::aarch64::inst::*; use crate::isa::aarch64::settings as aarch64_settings; use crate::machinst::lower::*; @@ -469,29 +468,7 @@ pub(crate) fn lower_insn_to_regs( } } - Opcode::Return => { - for (i, input) in inputs.iter().enumerate() { - // N.B.: according to the AArch64 ABI, the top bits of a register - // (above the bits for the value's type) are undefined, so we - // need not extend the return values. - let src_regs = put_input_in_regs(ctx, *input); - let retval_regs = ctx.retval(i); - - assert_eq!(src_regs.len(), retval_regs.len()); - let ty = ctx.input_ty(insn, i); - let (_, tys) = Inst::rc_for_type(ty)?; - - src_regs - .regs() - .iter() - .zip(retval_regs.regs().iter()) - .zip(tys.iter()) - .for_each(|((&src, &dst), &ty)| { - ctx.emit(Inst::gen_move(dst, src, ty)); - }); - } - // N.B.: the Ret itself is generated by the ABI. - } + Opcode::Return => implemented_in_isle(ctx), Opcode::Ifcmp | Opcode::Ffcmp => { // An Ifcmp/Ffcmp must always be seen as a use of a brif/brff or trueif/trueff @@ -577,52 +554,7 @@ pub(crate) fn lower_insn_to_regs( Opcode::SymbolValue => implemented_in_isle(ctx), - Opcode::Call | Opcode::CallIndirect => { - let caller_conv = ctx.abi().call_conv(); - let (mut abi, inputs) = match op { - Opcode::Call => { - let (extname, dist) = ctx.call_target(insn).unwrap(); - let extname = extname.clone(); - let sig = ctx.call_sig(insn).unwrap(); - assert!(inputs.len() == sig.params.len()); - assert!(outputs.len() == sig.returns.len()); - ( - AArch64Caller::from_func(sig, &extname, dist, caller_conv, flags)?, - &inputs[..], - ) - } - Opcode::CallIndirect => { - let ptr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); - let sig = ctx.call_sig(insn).unwrap(); - assert!(inputs.len() - 1 == sig.params.len()); - assert!(outputs.len() == sig.returns.len()); - ( - AArch64Caller::from_ptr(sig, ptr, op, caller_conv, flags)?, - &inputs[1..], - ) - } - _ => unreachable!(), - }; - - abi.emit_stack_pre_adjust(ctx); - assert!(inputs.len() == abi.num_args()); - let mut arg_regs = vec![]; - for input in inputs { - arg_regs.push(put_input_in_regs(ctx, *input)) - } - for (i, arg_regs) in arg_regs.iter().enumerate() { - abi.emit_copy_regs_to_buffer(ctx, i, *arg_regs); - } - for (i, arg_regs) in arg_regs.iter().enumerate() { - abi.emit_copy_regs_to_arg(ctx, i, *arg_regs); - } - abi.emit_call(ctx); - for (i, output) in outputs.iter().enumerate() { - let retval_regs = get_output_reg(ctx, *output); - abi.emit_copy_retval_to_regs(ctx, i, retval_regs); - } - abi.emit_stack_post_adjust(ctx); - } + Opcode::Call | Opcode::CallIndirect => implemented_in_isle(ctx), Opcode::GetPinnedReg => { let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle index bec494e220bf..172e6721476a 100644 --- a/cranelift/codegen/src/isa/s390x/inst.isle +++ b/cranelift/codegen/src/isa/s390x/inst.isle @@ -2940,8 +2940,8 @@ (rule (abi_ext_ty (ArgumentExtension.Sext) _) $I64) ;; Copy a return value to a set of registers. -(decl copy_to_regs (WritableValueRegs Value) Unit) -(rule (copy_to_regs (only_writable_reg reg) val @ (value_type ty)) +(decl s390x_copy_to_regs (WritableValueRegs Value) Unit) +(rule (s390x_copy_to_regs (only_writable_reg reg) val @ (value_type ty)) (emit_mov ty reg val)) diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle index 3848ce375eeb..7d15263a4ffb 100644 --- a/cranelift/codegen/src/isa/s390x/lower.isle +++ b/cranelift/codegen/src/isa/s390x/lower.isle @@ -4047,13 +4047,13 @@ ;;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (return args)) - (lower_return (range 0 (value_slice_len args)) args)) + (s390x_lower_return (range 0 (value_slice_len args)) args)) -(decl lower_return (Range ValueSlice) InstOutput) -(rule (lower_return (range_empty) _) (output_none)) -(rule (lower_return (range_unwrap head tail) args) - (let ((_ Unit (copy_to_regs (retval head) (value_slice_get args head)))) - (lower_return tail args))) +(decl s390x_lower_return (Range ValueSlice) InstOutput) +(rule (s390x_lower_return (range_empty) _) (output_none)) +(rule (s390x_lower_return (range_unwrap head tail) args) + (let ((_ Unit (s390x_copy_to_regs (retval head) (value_slice_get args head)))) + (s390x_lower_return tail args))) ;;;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index b8587a44adae..438e5980635f 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -22,7 +22,7 @@ use crate::{ isa::unwind::UnwindInst, isa::CallConv, machinst::abi_impl::ABIMachineSpec, - machinst::{InsnOutput, Lower, VCodeConstant, VCodeConstantData}, + machinst::{InsnOutput, Lower, MachInst, VCodeConstant, VCodeConstantData}, }; use regalloc2::PReg; use smallvec::{smallvec, SmallVec}; diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 0f491c42e121..dc83469712ab 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -1591,31 +1591,6 @@ ;;;; Helpers for Emitting Loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Generate a move between two registers. -(decl gen_move (Type WritableReg Reg) MInst) -(extern constructor gen_move gen_move) - -;; Copy a return value to a set of registers. -(decl copy_to_regs (WritableValueRegs Value) Unit) -(rule (copy_to_regs dsts val @ (value_type ty)) - (let ((srcs ValueRegs (put_in_regs val))) - (copy_to_regs_range ty (value_regs_range srcs) dsts srcs))) - -;; Helper for `copy_to_regs` that uses a range to index into the reg/value -;; vectors. Fails for the empty range. -(decl copy_to_regs_range (Type Range WritableValueRegs ValueRegs) Unit) - -(rule (copy_to_regs_range ty (range_singleton idx) dsts srcs) - (let ((dst WritableReg (writable_regs_get dsts idx)) - (src Reg (value_regs_get srcs idx))) - (emit (gen_move ty dst src)))) - -(rule (copy_to_regs_range ty (range_unwrap head tail) dsts srcs) - (let ((dst WritableReg (writable_regs_get dsts head)) - (src Reg (value_regs_get srcs head)) - (_ Unit (emit (gen_move ty dst src)))) - (copy_to_regs_range ty tail dsts srcs))) - ;; Helper for constructing a LoadExtName instruction. (decl load_ext_name (ExternalName i64) Reg) (rule (load_ext_name extname offset) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 3e863633c007..d9a65a7cf638 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -1441,13 +1441,6 @@ (rule (lower (return args)) (lower_return (range 0 (value_slice_len args)) args)) -(decl lower_return (Range ValueSlice) InstOutput) -(rule (lower_return (range_empty) _) (output_none)) -(rule (lower_return (range_unwrap head tail) args) - (let ((_ Unit (copy_to_regs (retval head) (value_slice_get args head)))) - (lower_return tail args))) - - ;;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (icmp cc a @ (value_type (fits_in_64 ty)) b)) diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 15daf707a9ba..c1ca8dce0bdd 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -92,6 +92,7 @@ pub(crate) fn lower_branch( impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { isle_prelude_methods!(); + isle_prelude_caller_methods!(X64ABIMachineSpec, X64Caller); #[inline] fn operand_size_of_type_32_64(&mut self, ty: Type) -> OperandSize { @@ -666,54 +667,6 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { MachAtomicRmwOp::from(*op) } - #[inline] - fn gen_move(&mut self, ty: Type, dst: WritableReg, src: Reg) -> MInst { - MInst::gen_move(dst, src, ty) - } - - fn gen_call( - &mut self, - sig_ref: SigRef, - extname: ExternalName, - dist: RelocDistance, - args @ (inputs, off): ValueSlice, - ) -> InstOutput { - let caller_conv = self.lower_ctx.abi().call_conv(); - let sig = &self.lower_ctx.dfg().signatures[sig_ref]; - let num_rets = sig.returns.len(); - let abi = ABISig::from_func_sig::(sig, self.flags).unwrap(); - let caller = X64Caller::from_func(sig, &extname, dist, caller_conv, self.flags).unwrap(); - - assert_eq!( - inputs.len(&self.lower_ctx.dfg().value_lists) - off, - sig.params.len() - ); - - self.gen_call_common(abi, num_rets, caller, args) - } - - fn gen_call_indirect( - &mut self, - sig_ref: SigRef, - val: Value, - args @ (inputs, off): ValueSlice, - ) -> InstOutput { - let caller_conv = self.lower_ctx.abi().call_conv(); - let ptr = self.put_in_reg(val); - let sig = &self.lower_ctx.dfg().signatures[sig_ref]; - let num_rets = sig.returns.len(); - let abi = ABISig::from_func_sig::(sig, self.flags).unwrap(); - let caller = - X64Caller::from_ptr(sig, ptr, Opcode::CallIndirect, caller_conv, self.flags).unwrap(); - - assert_eq!( - inputs.len(&self.lower_ctx.dfg().value_lists) - off, - sig.params.len() - ); - - self.gen_call_common(abi, num_rets, caller, args) - } - #[inline] fn preg_rbp(&mut self) -> PReg { regs::rbp().to_real_reg().unwrap().into() @@ -1020,63 +973,7 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { } impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> { - fn abi_arg_slot_regs(&mut self, arg: &ABIArg) -> Option { - match arg { - &ABIArg::Slots { ref slots, .. } => match slots.len() { - 1 => { - let a = self.temp_writable_reg(slots[0].get_type()); - Some(WritableValueRegs::one(a)) - } - 2 => { - let a = self.temp_writable_reg(slots[0].get_type()); - let b = self.temp_writable_reg(slots[1].get_type()); - Some(WritableValueRegs::two(a, b)) - } - _ => panic!("Expected to see one or two slots only from {:?}", arg), - }, - _ => None, - } - } - - fn gen_call_common( - &mut self, - abi: ABISig, - num_rets: usize, - mut caller: X64Caller, - (inputs, off): ValueSlice, - ) -> InstOutput { - caller.emit_stack_pre_adjust(self.lower_ctx); - - assert_eq!( - inputs.len(&self.lower_ctx.dfg().value_lists) - off, - abi.num_args() - ); - let mut arg_regs = vec![]; - for i in 0..abi.num_args() { - let input = inputs - .get(off + i, &self.lower_ctx.dfg().value_lists) - .unwrap(); - arg_regs.push(self.lower_ctx.put_value_in_regs(input)); - } - for (i, arg_regs) in arg_regs.iter().enumerate() { - caller.emit_copy_regs_to_buffer(self.lower_ctx, i, *arg_regs); - } - for (i, arg_regs) in arg_regs.iter().enumerate() { - caller.emit_copy_regs_to_arg(self.lower_ctx, i, *arg_regs); - } - caller.emit_call(self.lower_ctx); - - let mut outputs = InstOutput::new(); - for i in 0..num_rets { - let ret = abi.get_ret(i); - let retval_regs = self.abi_arg_slot_regs(&ret).unwrap(); - caller.emit_copy_retval_to_regs(self.lower_ctx, i, retval_regs.clone()); - outputs.push(valueregs::non_writable_value_regs(retval_regs)); - } - caller.emit_stack_post_adjust(self.lower_ctx); - - outputs - } + isle_prelude_method_helpers!(X64Caller); } // Since x64 doesn't have 8x16 shifts and we must use a 16x8 shift instead, we diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index a6a3a39657f4..1c9d5429bd1d 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -988,6 +988,129 @@ macro_rules! isle_prelude_methods { fn preg_to_reg(&mut self, preg: PReg) -> Reg { preg.into() } + + #[inline] + fn gen_move(&mut self, ty: Type, dst: WritableReg, src: Reg) -> MInst { + MInst::gen_move(dst, src, ty) + } + }; +} + +/// Helpers specifically for machines that use ABICaller. +#[macro_export] +#[doc(hidden)] +macro_rules! isle_prelude_caller_methods { + ($abispec:ty, $abicaller:ty) => { + fn gen_call( + &mut self, + sig_ref: SigRef, + extname: ExternalName, + dist: RelocDistance, + args @ (inputs, off): ValueSlice, + ) -> InstOutput { + let caller_conv = self.lower_ctx.abi().call_conv(); + let sig = &self.lower_ctx.dfg().signatures[sig_ref]; + let num_rets = sig.returns.len(); + let abi = ABISig::from_func_sig::<$abispec>(sig, self.flags).unwrap(); + let caller = + <$abicaller>::from_func(sig, &extname, dist, caller_conv, self.flags).unwrap(); + + assert_eq!( + inputs.len(&self.lower_ctx.dfg().value_lists) - off, + sig.params.len() + ); + + self.gen_call_common(abi, num_rets, caller, args) + } + + fn gen_call_indirect( + &mut self, + sig_ref: SigRef, + val: Value, + args @ (inputs, off): ValueSlice, + ) -> InstOutput { + let caller_conv = self.lower_ctx.abi().call_conv(); + let ptr = self.put_in_reg(val); + let sig = &self.lower_ctx.dfg().signatures[sig_ref]; + let num_rets = sig.returns.len(); + let abi = ABISig::from_func_sig::<$abispec>(sig, self.flags).unwrap(); + let caller = + <$abicaller>::from_ptr(sig, ptr, Opcode::CallIndirect, caller_conv, self.flags) + .unwrap(); + + assert_eq!( + inputs.len(&self.lower_ctx.dfg().value_lists) - off, + sig.params.len() + ); + + self.gen_call_common(abi, num_rets, caller, args) + } + }; +} + +/// Helpers for the above ISLE prelude implementations. Meant to go +/// inside the `impl` for the context type, not the trait impl. +#[macro_export] +#[doc(hidden)] +macro_rules! isle_prelude_method_helpers { + ($abicaller:ty) => { + fn gen_call_common( + &mut self, + abi: ABISig, + num_rets: usize, + mut caller: $abicaller, + (inputs, off): ValueSlice, + ) -> InstOutput { + caller.emit_stack_pre_adjust(self.lower_ctx); + + assert_eq!( + inputs.len(&self.lower_ctx.dfg().value_lists) - off, + abi.num_args() + ); + let mut arg_regs = vec![]; + for i in 0..abi.num_args() { + let input = inputs + .get(off + i, &self.lower_ctx.dfg().value_lists) + .unwrap(); + arg_regs.push(self.lower_ctx.put_value_in_regs(input)); + } + for (i, arg_regs) in arg_regs.iter().enumerate() { + caller.emit_copy_regs_to_buffer(self.lower_ctx, i, *arg_regs); + } + for (i, arg_regs) in arg_regs.iter().enumerate() { + caller.emit_copy_regs_to_arg(self.lower_ctx, i, *arg_regs); + } + caller.emit_call(self.lower_ctx); + + let mut outputs = InstOutput::new(); + for i in 0..num_rets { + let ret = abi.get_ret(i); + let retval_regs = self.abi_arg_slot_regs(&ret).unwrap(); + caller.emit_copy_retval_to_regs(self.lower_ctx, i, retval_regs.clone()); + outputs.push(valueregs::non_writable_value_regs(retval_regs)); + } + caller.emit_stack_post_adjust(self.lower_ctx); + + outputs + } + + fn abi_arg_slot_regs(&mut self, arg: &ABIArg) -> Option { + match arg { + &ABIArg::Slots { ref slots, .. } => match slots.len() { + 1 => { + let a = self.temp_writable_reg(slots[0].get_type()); + Some(WritableValueRegs::one(a)) + } + 2 => { + let a = self.temp_writable_reg(slots[0].get_type()); + let b = self.temp_writable_reg(slots[1].get_type()); + Some(WritableValueRegs::two(a, b)) + } + _ => panic!("Expected to see one or two slots only from {:?}", arg), + }, + _ => None, + } + } }; } diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index d03faae47235..eec970d87b9a 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -11,8 +11,8 @@ use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit}; use crate::ir::{ types::{FFLAGS, IFLAGS}, ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, Function, GlobalValue, - GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, Signature, Type, Value, - ValueDef, ValueLabelAssignments, ValueLabelStart, + GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, Type, Value, ValueDef, + ValueLabelAssignments, ValueLabelStart, }; use crate::ir::{ExternalName, RelSourceLoc}; use crate::machinst::{ @@ -1026,34 +1026,6 @@ impl<'func, I: VCodeInst> Lower<'func, I> { &self.f.dfg[ir_inst] } - /// Get the target for a call instruction, as an `ExternalName`. Returns a tuple - /// providing this name and the "relocation distance", i.e., whether the backend - /// can assume the target will be "nearby" (within some small offset) or an - /// arbitrary address. (This comes from the `colocated` bit in the CLIF.) - pub fn call_target<'b>(&'b self, ir_inst: Inst) -> Option<(&'b ExternalName, RelocDistance)> { - match &self.f.dfg[ir_inst] { - &InstructionData::Call { func_ref, .. } - | &InstructionData::FuncAddr { func_ref, .. } => { - let funcdata = &self.f.dfg.ext_funcs[func_ref]; - let dist = funcdata.reloc_distance(); - Some((&funcdata.name, dist)) - } - _ => None, - } - } - - /// Get the signature for a call or call-indirect instruction. - pub fn call_sig<'b>(&'b self, ir_inst: Inst) -> Option<&'b Signature> { - match &self.f.dfg[ir_inst] { - &InstructionData::Call { func_ref, .. } => { - let funcdata = &self.f.dfg.ext_funcs[func_ref]; - Some(&self.f.dfg.signatures[funcdata.signature]) - } - &InstructionData::CallIndirect { sig_ref, .. } => Some(&self.f.dfg.signatures[sig_ref]), - _ => None, - } - } - /// Get the symbol name, relocation distance estimate, and offset for a /// symbol_value instruction. pub fn symbol_value<'b>( diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 5eb7a6c5dd0f..c001172af884 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -957,6 +957,37 @@ (decl real_reg_to_writable_reg (RealReg) WritableReg) (extern constructor real_reg_to_writable_reg real_reg_to_writable_reg) +;; Generate a move between two registers. +(decl gen_move (Type WritableReg Reg) MInst) +(extern constructor gen_move gen_move) + +;; Copy a return value to a set of registers. +(decl copy_to_regs (WritableValueRegs Value) Unit) +(rule (copy_to_regs dsts val @ (value_type ty)) + (let ((srcs ValueRegs (put_in_regs val))) + (copy_to_regs_range ty (value_regs_range srcs) dsts srcs))) + +;; Helper for `copy_to_regs` that uses a range to index into the reg/value +;; vectors. Fails for the empty range. +(decl copy_to_regs_range (Type Range WritableValueRegs ValueRegs) Unit) + +(rule (copy_to_regs_range ty (range_singleton idx) dsts srcs) + (let ((dst WritableReg (writable_regs_get dsts idx)) + (src Reg (value_regs_get srcs idx))) + (emit (gen_move ty dst src)))) + +(rule (copy_to_regs_range ty (range_unwrap head tail) dsts srcs) + (let ((dst WritableReg (writable_regs_get dsts head)) + (src Reg (value_regs_get srcs head)) + (_ Unit (emit (gen_move ty dst src)))) + (copy_to_regs_range ty tail dsts srcs))) + +(decl lower_return (Range ValueSlice) InstOutput) +(rule (lower_return (range_empty) _) (output_none)) +(rule (lower_return (range_unwrap head tail) args) + (let ((_ Unit (copy_to_regs (retval head) (value_slice_get args head)))) + (lower_return tail args))) + ;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (convert Inst Value def_inst) diff --git a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif index 45b7a4f60922..bb1a853f90ee 100644 --- a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif +++ b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif @@ -14,8 +14,8 @@ block0(v0: i64): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase(%g) + 0 -; blr x5 +; ldr x4, 8 ; b 12 ; data TestCase(%g) + 0 +; blr x4 ; ldp fp, lr, [sp], #16 ; autiasp ; ret diff --git a/cranelift/filetests/filetests/isa/aarch64/call.clif b/cranelift/filetests/filetests/isa/aarch64/call.clif index b295c9509419..0f2837f073cd 100644 --- a/cranelift/filetests/filetests/isa/aarch64/call.clif +++ b/cranelift/filetests/filetests/isa/aarch64/call.clif @@ -14,8 +14,8 @@ block0(v0: i64): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase(%g) + 0 -; blr x5 +; ldr x4, 8 ; b 12 ; data TestCase(%g) + 0 +; blr x4 ; ldp fp, lr, [sp], #16 ; ret @@ -30,8 +30,8 @@ block0(v0: i32): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase(%g) + 0 -; blr x5 +; ldr x4, 8 ; b 12 ; data TestCase(%g) + 0 +; blr x4 ; ldp fp, lr, [sp], #16 ; ret @@ -54,8 +54,8 @@ block0(v0: i32): ; stp fp, lr, [sp, #-16]! ; mov fp, sp ; block0: -; ldr x5, 8 ; b 12 ; data TestCase(%g) + 0 -; blr x5 +; ldr x4, 8 ; b 12 ; data TestCase(%g) + 0 +; blr x4 ; ldp fp, lr, [sp], #16 ; ret @@ -91,8 +91,8 @@ block0(v0: i8): ; movz x6, #42 ; movz x7, #42 ; strb w15, [sp] -; ldr x15, 8 ; b 12 ; data TestCase(%g) + 0 -; blr x15 +; ldr x14, 8 ; b 12 ; data TestCase(%g) + 0 +; blr x14 ; add sp, sp, #16 ; virtual_sp_offset_adjust -16 ; ldp fp, lr, [sp], #16 @@ -140,26 +140,26 @@ block0: ; mov fp, sp ; sub sp, sp, #48 ; block0: -; ldr x9, 8 ; b 12 ; data TestCase(%g0) + 0 +; ldr x8, 8 ; b 12 ; data TestCase(%g0) + 0 +; blr x8 +; str q0, [sp, #32] +; ldr x9, 8 ; b 12 ; data TestCase(%g1) + 0 ; blr x9 -; str q0, [sp] -; ldr x11, 8 ; b 12 ; data TestCase(%g1) + 0 -; blr x11 ; str q0, [sp, #16] -; ldr x13, 8 ; b 12 ; data TestCase(%g1) + 0 -; blr x13 -; str q0, [sp, #32] -; ldr x15, 8 ; b 12 ; data TestCase(%g2) + 0 -; blr x15 -; ldr q0, [sp] -; ldr x1, 8 ; b 12 ; data TestCase(%g3) + 0 -; blr x1 -; ldr q0, [sp, #16] -; ldr x3, 8 ; b 12 ; data TestCase(%g4) + 0 -; blr x3 +; ldr x10, 8 ; b 12 ; data TestCase(%g1) + 0 +; blr x10 +; str q0, [sp] +; ldr x12, 8 ; b 12 ; data TestCase(%g2) + 0 +; blr x12 ; ldr q0, [sp, #32] -; ldr x5, 8 ; b 12 ; data TestCase(%g4) + 0 -; blr x5 +; ldr x14, 8 ; b 12 ; data TestCase(%g3) + 0 +; blr x14 +; ldr q0, [sp, #16] +; ldr x0, 8 ; b 12 ; data TestCase(%g4) + 0 +; blr x0 +; ldr q0, [sp] +; ldr x2, 8 ; b 12 ; data TestCase(%g4) + 0 +; blr x2 ; add sp, sp, #48 ; ldp fp, lr, [sp], #16 ; ret @@ -184,26 +184,26 @@ block0: ; mov fp, sp ; sub sp, sp, #48 ; block0: +; ldr x8, 8 ; b 12 ; data TestCase(%g0) + 0 +; blr x8 +; str q0, [sp, #32] ; ldr x9, 8 ; b 12 ; data TestCase(%g0) + 0 ; blr x9 -; str q0, [sp] -; ldr x11, 8 ; b 12 ; data TestCase(%g0) + 0 -; blr x11 ; str q0, [sp, #16] -; ldr x13, 8 ; b 12 ; data TestCase(%g0) + 0 -; blr x13 -; str q0, [sp, #32] -; ldr x15, 8 ; b 12 ; data TestCase(%g1) + 0 -; blr x15 -; ldr q0, [sp] -; ldr x1, 8 ; b 12 ; data TestCase(%g2) + 0 -; blr x1 -; ldr q0, [sp, #16] -; ldr x3, 8 ; b 12 ; data TestCase(%g2) + 0 -; blr x3 +; ldr x10, 8 ; b 12 ; data TestCase(%g0) + 0 +; blr x10 +; str q0, [sp] +; ldr x12, 8 ; b 12 ; data TestCase(%g1) + 0 +; blr x12 ; ldr q0, [sp, #32] -; ldr x5, 8 ; b 12 ; data TestCase(%g2) + 0 -; blr x5 +; ldr x14, 8 ; b 12 ; data TestCase(%g2) + 0 +; blr x14 +; ldr q0, [sp, #16] +; ldr x0, 8 ; b 12 ; data TestCase(%g2) + 0 +; blr x0 +; ldr q0, [sp] +; ldr x2, 8 ; b 12 ; data TestCase(%g2) + 0 +; blr x2 ; add sp, sp, #48 ; ldp fp, lr, [sp], #16 ; ret @@ -232,26 +232,26 @@ block0: ; mov fp, sp ; sub sp, sp, #48 ; block0: -; ldr x9, 8 ; b 12 ; data TestCase(%g0) + 0 +; ldr x8, 8 ; b 12 ; data TestCase(%g0) + 0 +; blr x8 +; str q0, [sp, #32] +; ldr x9, 8 ; b 12 ; data TestCase(%g1) + 0 ; blr x9 -; str q0, [sp] -; ldr x11, 8 ; b 12 ; data TestCase(%g1) + 0 -; blr x11 ; str q0, [sp, #16] -; ldr x13, 8 ; b 12 ; data TestCase(%g2) + 0 -; blr x13 -; str q0, [sp, #32] -; ldr x15, 8 ; b 12 ; data TestCase(%g3) + 0 -; blr x15 -; ldr q0, [sp] -; ldr x1, 8 ; b 12 ; data TestCase(%g4) + 0 -; blr x1 -; ldr q0, [sp, #16] -; ldr x3, 8 ; b 12 ; data TestCase(%g5) + 0 -; blr x3 +; ldr x10, 8 ; b 12 ; data TestCase(%g2) + 0 +; blr x10 +; str q0, [sp] +; ldr x12, 8 ; b 12 ; data TestCase(%g3) + 0 +; blr x12 ; ldr q0, [sp, #32] -; ldr x5, 8 ; b 12 ; data TestCase(%g6) + 0 -; blr x5 +; ldr x14, 8 ; b 12 ; data TestCase(%g4) + 0 +; blr x14 +; ldr q0, [sp, #16] +; ldr x0, 8 ; b 12 ; data TestCase(%g5) + 0 +; blr x0 +; ldr q0, [sp] +; ldr x2, 8 ; b 12 ; data TestCase(%g6) + 0 +; blr x2 ; add sp, sp, #48 ; ldp fp, lr, [sp], #16 ; ret @@ -283,8 +283,8 @@ block0(v0: i64): ; movz x0, #42 ; movz x2, #42 ; mov x1, x7 -; ldr x10, 8 ; b 12 ; data TestCase(%f11) + 0 -; blr x10 +; ldr x9, 8 ; b 12 ; data TestCase(%f11) + 0 +; blr x9 ; ldp fp, lr, [sp], #16 ; ret @@ -315,8 +315,8 @@ block0(v0: i64): ; movz x3, #42 ; movz x0, #42 ; mov x2, x7 -; ldr x10, 8 ; b 12 ; data TestCase(%f12) + 0 -; blr x10 +; ldr x9, 8 ; b 12 ; data TestCase(%f12) + 0 +; blr x9 ; ldp fp, lr, [sp], #16 ; ret @@ -347,8 +347,8 @@ block0(v0: i64): ; movz x2, #42 ; movz x0, #42 ; mov x1, x7 -; ldr x10, 8 ; b 12 ; data TestCase(%f13) + 0 -; blr x10 +; ldr x9, 8 ; b 12 ; data TestCase(%f13) + 0 +; blr x9 ; ldp fp, lr, [sp], #16 ; ret @@ -477,8 +477,8 @@ block0(v0: i64): ; mov fp, sp ; block0: ; mov x8, x0 -; ldr x5, 8 ; b 12 ; data TestCase(%g) + 0 -; blr x5 +; ldr x4, 8 ; b 12 ; data TestCase(%g) + 0 +; blr x4 ; mov x0, x8 ; ldp fp, lr, [sp], #16 ; ret diff --git a/cranelift/filetests/filetests/isa/aarch64/constants.clif b/cranelift/filetests/filetests/isa/aarch64/constants.clif index 130ecdd4756b..a6a7a95549b6 100644 --- a/cranelift/filetests/filetests/isa/aarch64/constants.clif +++ b/cranelift/filetests/filetests/isa/aarch64/constants.clif @@ -164,7 +164,7 @@ block0: } ; block0: -; orr x0, xzr, #4294967295 +; movn x0, #0 ; ret function %f() -> i32 { diff --git a/cranelift/filetests/filetests/isa/aarch64/reftypes.clif b/cranelift/filetests/filetests/isa/aarch64/reftypes.clif index ef58774bfc3c..474629ed1c88 100644 --- a/cranelift/filetests/filetests/isa/aarch64/reftypes.clif +++ b/cranelift/filetests/filetests/isa/aarch64/reftypes.clif @@ -69,13 +69,13 @@ block3(v7: r64, v8: r64): ; block0: ; str x1, [sp, #16] ; str x0, [sp, #8] -; ldr x3, 8 ; b 12 ; data TestCase(%f) + 0 -; blr x3 -; mov x9, sp +; ldr x2, 8 ; b 12 ; data TestCase(%f) + 0 +; blr x2 +; mov x8, sp ; ldr x11, [sp, #8] -; str x11, [x9] -; and w7, w0, #1 -; cbz x7, label1 ; b label3 +; str x11, [x8] +; and w6, w0, #1 +; cbz x6, label1 ; b label3 ; block1: ; b label2 ; block2: @@ -89,8 +89,8 @@ block3(v7: r64, v8: r64): ; ldr x1, [sp, #16] ; b label5 ; block5: -; mov x4, sp -; ldr x2, [x4] +; mov x3, sp +; ldr x2, [x3] ; add sp, sp, #32 ; ldp fp, lr, [sp], #16 ; ret