Skip to content

Commit

Permalink
x64: Lower tlsvalue, sqmul_round_sat, and uunarrow in ISLE (#4793)
Browse files Browse the repository at this point in the history 
Lower tlsvalue, sqmul_round_sat, and uunarrow in ISLE.
  • Loading branch information
@elliottt
elliottt authored Aug 26, 2022
1 parent 8e8dfdf commit 25d960f
Show file tree
Hide file tree
Showing 11 changed files with 287 additions and 205 deletions.
65 changes: 59 additions & 6 deletions cranelift/codegen/src/isa/x64/inst.isle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,16 +486,19 @@
(XmmUninitializedValue (dst WritableXmm))

;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol
;; in `rax`.
(ElfTlsGetAddr (symbol ExternalName))
;; `dst`, which is constrained to `rax`.
(ElfTlsGetAddr (symbol ExternalName)
(dst WritableGpr))

;; A Mach-O TLS symbol access. Returns address of the TLS symbol in
;; `rax`.
(MachOTlsGetAddr (symbol ExternalName))
;; `dst`, which is constrained to `rax`.
(MachOTlsGetAddr (symbol ExternalName)
(dst WritableGpr))

;; A Coff TLS symbol access. Returns address of the TLS symbol in
;; `rax`.
(CoffTlsGetAddr (symbol ExternalName))
;; `dst`, which is constrained to `rax`.
(CoffTlsGetAddr (symbol ExternalName)
(dst WritableGpr))

;; An unwind pseudoinstruction describing the state of the machine at
;; this program point.
Expand Down Expand Up @@ -2275,6 +2278,11 @@
(rule (x64_pmulhw src1 src2)
(xmm_rm_r $I16X8 (SseOpcode.Pmulhw) src1 src2))

;; Helper for creating `pmulhrsw` instructions.
(decl x64_pmulhrsw (Xmm XmmMem) Xmm)
(rule (x64_pmulhrsw src1 src2)
(xmm_rm_r $I16X8 (SseOpcode.Pmulhrsw) src1 src2))

;; Helper for creating `pmulhuw` instructions.
(decl x64_pmulhuw (Xmm XmmMem) Xmm)
(rule (x64_pmulhuw src1 src2)
Expand Down Expand Up @@ -2683,6 +2691,15 @@
dst))))
dst))

;; Helper for creating `shufps` instructions.
(decl x64_shufps (Xmm XmmMem u8) Xmm)
(rule (x64_shufps src1 src2 byte)
(xmm_rm_r_imm (SseOpcode.Shufps)
src1
src2
byte
(OperandSize.Size32)))

;; Helper for creating `MInst.XmmUnaryRmR` instructions.
(decl xmm_unary_rm_r (SseOpcode XmmMem) Xmm)
(rule (xmm_unary_rm_r op src)
Expand Down Expand Up @@ -3733,6 +3750,42 @@
(decl swizzle_zero_mask () VCodeConstant)
(extern constructor swizzle_zero_mask swizzle_zero_mask)

;;;; TLS Values ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; Helper for emitting ElfTlsGetAddr.
(decl elf_tls_get_addr (ExternalName) Gpr)
(rule (elf_tls_get_addr name)
(let ((dst WritableGpr (temp_writable_gpr))
(_ Unit (emit (MInst.ElfTlsGetAddr name dst))))
dst))

;; Helper for emitting MachOTlsGetAddr.
(decl macho_tls_get_addr (ExternalName) Gpr)
(rule (macho_tls_get_addr name)
(let ((dst WritableGpr (temp_writable_gpr))
(_ Unit (emit (MInst.MachOTlsGetAddr name dst))))
dst))

;; Helper for emitting CoffTlsGetAddr.
(decl coff_tls_get_addr (ExternalName) Gpr)
(rule (coff_tls_get_addr name)
(let ((dst WritableGpr (temp_writable_gpr))
(_ Unit (emit (MInst.CoffTlsGetAddr name dst))))
dst))

;;;; sqmul_round_sat ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(decl sqmul_round_sat_mask () VCodeConstant)
(extern constructor sqmul_round_sat_mask sqmul_round_sat_mask)

;;;; uunarrow ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(decl uunarrow_umax_mask () VCodeConstant)
(extern constructor uunarrow_umax_mask uunarrow_umax_mask)

(decl uunarrow_uint_mask () VCodeConstant)
(extern constructor uunarrow_uint_mask uunarrow_uint_mask)

;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(convert Gpr InstOutput output_gpr)
Expand Down
15 changes: 12 additions & 3 deletions cranelift/codegen/src/isa/x64/inst/emit.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2915,7 +2915,10 @@ pub(crate) fn emit(
}
}

Inst::ElfTlsGetAddr { ref symbol } => {
Inst::ElfTlsGetAddr { ref symbol, dst } => {
let dst = allocs.next(dst.to_reg().to_reg());
debug_assert_eq!(dst, regs::rax());

// N.B.: Must be exactly this byte sequence; the linker requires it,
// because it must know how to rewrite the bytes.

Expand All @@ -2941,7 +2944,10 @@ pub(crate) fn emit(
sink.put4(0); // offset
}

Inst::MachOTlsGetAddr { ref symbol } => {
Inst::MachOTlsGetAddr { ref symbol, dst } => {
let dst = allocs.next(dst.to_reg().to_reg());
debug_assert_eq!(dst, regs::rax());

// movq gv@tlv(%rip), %rdi
sink.put1(0x48); // REX.w
sink.put1(0x8b); // MOV
Expand All @@ -2954,7 +2960,10 @@ pub(crate) fn emit(
sink.put1(0x17);
}

Inst::CoffTlsGetAddr { ref symbol } => {
Inst::CoffTlsGetAddr { ref symbol, dst } => {
let dst = allocs.next(dst.to_reg().to_reg());
debug_assert_eq!(dst, regs::rax());

// See: https://gcc.godbolt.org/z/M8or9x6ss
// And: https://github.com/bjorn3/rustc_codegen_cranelift/issues/388#issuecomment-532930282

Expand Down
21 changes: 21 additions & 0 deletions cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,24 @@ impl Inst {
let dst = WritableGpr::from_writable_reg(dst).unwrap();
Inst::Setcc { cc, dst }
}

fn xmm_rm_r_imm(
op: SseOpcode,
src: RegMem,
dst: Writable<Reg>,
imm: u8,
size: OperandSize,
) -> Inst {
debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
Inst::XmmRmRImm {
op,
src1: dst.to_reg(),
src2: src,
dst,
imm,
size,
}
}
}

#[test]
Expand Down Expand Up @@ -4738,6 +4756,7 @@ fn test_x64_emit() {
insns.push((
Inst::ElfTlsGetAddr {
symbol: ExternalName::User(UserExternalNameRef::new(0)),
dst: WritableGpr::from_writable_reg(w_rax).unwrap(),
},
"66488D3D00000000666648E800000000",
"%rax = elf_tls_get_addr User(userextname0)",
Expand All @@ -4746,6 +4765,7 @@ fn test_x64_emit() {
insns.push((
Inst::MachOTlsGetAddr {
symbol: ExternalName::User(UserExternalNameRef::new(0)),
dst: WritableGpr::from_writable_reg(w_rax).unwrap(),
},
"488B3D00000000FF17",
"%rax = macho_tls_get_addr User(userextname0)",
Expand All @@ -4754,6 +4774,7 @@ fn test_x64_emit() {
insns.push((
Inst::CoffTlsGetAddr {
symbol: ExternalName::User(UserExternalNameRef::new(0)),
dst: WritableGpr::from_writable_reg(w_rax).unwrap(),
},
"8B050000000065488B0C2558000000488B04C1488D8000000000",
"%rax = coff_tls_get_addr User(userextname0)",
Expand Down
47 changes: 13 additions & 34 deletions cranelift/codegen/src/isa/x64/inst/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,12 +263,6 @@ impl Inst {
Inst::MovRR { size, src, dst }
}

pub(crate) fn xmm_load_const(src: VCodeConstant, dst: Writable<Reg>, ty: Type) -> Inst {
debug_assert!(dst.to_reg().class() == RegClass::Float);
debug_assert!(ty.is_vector() && ty.bits() == 128);
Inst::XmmLoadConst { src, dst, ty }
}

/// Convenient helper for unary float operations.
pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::Float);
Expand Down Expand Up @@ -377,24 +371,6 @@ impl Inst {
}
}

pub(crate) fn xmm_rm_r_imm(
op: SseOpcode,
src: RegMem,
dst: Writable<Reg>,
imm: u8,
size: OperandSize,
) -> Inst {
debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
Inst::XmmRmRImm {
op,
src1: dst.to_reg(),
src2: src,
dst,
imm,
size,
}
}

pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
src.assert_regclass_is(RegClass::Int);
debug_assert!(dst.to_reg().class() == RegClass::Int);
Expand Down Expand Up @@ -1544,16 +1520,19 @@ impl PrettyPrint for Inst {

Inst::Ud2 { trap_code } => format!("ud2 {}", trap_code),

Inst::ElfTlsGetAddr { ref symbol } => {
format!("%rax = elf_tls_get_addr {:?}", symbol)
Inst::ElfTlsGetAddr { ref symbol, dst } => {
let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
format!("{} = elf_tls_get_addr {:?}", dst, symbol)
}

Inst::MachOTlsGetAddr { ref symbol } => {
format!("%rax = macho_tls_get_addr {:?}", symbol)
Inst::MachOTlsGetAddr { ref symbol, dst } => {
let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
format!("{} = macho_tls_get_addr {:?}", dst, symbol)
}

Inst::CoffTlsGetAddr { ref symbol } => {
format!("%rax = coff_tls_get_addr {:?}", symbol)
Inst::CoffTlsGetAddr { ref symbol, dst } => {
let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
format!("{} = coff_tls_get_addr {:?}", dst, symbol)
}

Inst::Unwind { inst } => {
Expand Down Expand Up @@ -1994,8 +1973,8 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
// No registers are used.
}

Inst::ElfTlsGetAddr { .. } | Inst::MachOTlsGetAddr { .. } => {
collector.reg_def(Writable::from_reg(regs::rax()));
Inst::ElfTlsGetAddr { dst, .. } | Inst::MachOTlsGetAddr { dst, .. } => {
collector.reg_fixed_def(dst.to_writable_reg(), regs::rax());
// All caller-saves are clobbered.
//
// We use the SysV calling convention here because the
Expand All @@ -2007,12 +1986,12 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
collector.reg_clobbers(clobbers);
}

Inst::CoffTlsGetAddr { .. } => {
Inst::CoffTlsGetAddr { dst, .. } => {
// We also use the gs register. But that register is not allocatable by the
// register allocator, so we don't need to mark it as used here.

// We use %rax to set the address
collector.reg_def(Writable::from_reg(regs::rax()));
collector.reg_fixed_def(dst.to_writable_reg(), regs::rax());

// We use %rcx as a temporary variable to load the _tls_index
collector.reg_def(Writable::from_reg(regs::rcx()));
Expand Down
63 changes: 63 additions & 0 deletions cranelift/codegen/src/isa/x64/lower.isle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3694,3 +3694,66 @@
(lo Reg (value_regs_get regs 0))
(hi Reg (value_regs_get regs 1)))
(output_pair lo hi)))

;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(rule (lower (tls_value (symbol_value_data name _ _)))
(if (tls_model_is_elf_gd))
(elf_tls_get_addr name))

(rule (lower (tls_value (symbol_value_data name _ _)))
(if (tls_model_is_macho))
(macho_tls_get_addr name))

(rule (lower (tls_value (symbol_value_data name _ _)))
(if (tls_model_is_coff))
(coff_tls_get_addr name))

;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(rule (lower (sqmul_round_sat qx @ (value_type $I16X8) qy))
(let ((src1 Xmm qx)
(src2 Xmm qy)

(mask Xmm (x64_xmm_load_const $I16X8 (sqmul_round_sat_mask)))
(dst Xmm (x64_pmulhrsw src1 src2))
(cmp Xmm (x64_pcmpeqw mask dst)))
(x64_pxor dst cmp)))

;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

;; TODO: currently we only lower a special case of `uunarrow` needed to support
;; the translation of wasm's i32x4.trunc_sat_f64x2_u_zero operation.
;; https://github.com/bytecodealliance/wasmtime/issues/4791
;;
;; y = i32x4.trunc_sat_f64x2_u_zero(x) is lowered to:
;; MOVAPD xmm_y, xmm_x
;; XORPD xmm_tmp, xmm_tmp
;; MAXPD xmm_y, xmm_tmp
;; MINPD xmm_y, [wasm_f64x2_splat(4294967295.0)]
;; ROUNDPD xmm_y, xmm_y, 0x0B
;; ADDPD xmm_y, [wasm_f64x2_splat(0x1.0p+52)]
;; SHUFPS xmm_y, xmm_xmp, 0x88
(rule (lower (uunarrow (fcvt_to_uint_sat src @ (value_type $F64X2))
(vconst (u128_from_constant 0))))
(let ((src Xmm src)

;; MOVAPD xmm_y, xmm_x
;; XORPD xmm_tmp, xmm_tmp
(zeros Xmm (x64_xorpd src src))
(dst Xmm (x64_maxpd src zeros))

(umax_mask Xmm (x64_xmm_load_const $F64X2 (uunarrow_umax_mask)))

;; MINPD xmm_y, [wasm_f64x2_splat(4294967295.0)]
(dst Xmm (x64_minpd dst umax_mask))

;; ROUNDPD xmm_y, xmm_y, 0x0B
(dst Xmm (x64_roundpd dst (RoundImm.RoundZero)))

;; ADDPD xmm_y, [wasm_f64x2_splat(0x1.0p+52)]
(uint_mask Xmm (x64_xmm_load_const $F64X2 (uunarrow_uint_mask)))
(dst Xmm (x64_addpd dst uint_mask)))

;; SHUFPS xmm_y, xmm_xmp, 0x88
(x64_shufps dst zeros 0x88)))
Loading

0 comments on commit 25d960f

Please sign in to comment.