diff --git a/build.rs b/build.rs
index 4918c3466e43..18e4c80e7ca4 100644
--- a/build.rs
+++ b/build.rs
@@ -171,9 +171,9 @@ fn write_testsuite_tests(
fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool {
match strategy {
"Cranelift" => match (testsuite, testname) {
- // No simd support yet for s390x.
- ("simd", _) if platform_is_s390x() => return true,
- _ if platform_is_s390x() && testname.starts_with("simd") => return true,
+ // FIXME: These tests fail under qemu due to a qemu bug.
+ (_, "simd_f32x4_pmin_pmax") if platform_is_s390x() => return true,
+ (_, "simd_f64x2_pmin_pmax") if platform_is_s390x() => return true,
_ => {}
},
_ => panic!("unrecognized strategy"),
diff --git a/cranelift/codegen/src/data_value.rs b/cranelift/codegen/src/data_value.rs
index 13aa23767499..e2b6d5aba395 100644
--- a/cranelift/codegen/src/data_value.rs
+++ b/cranelift/codegen/src/data_value.rs
@@ -89,7 +89,7 @@ impl DataValue {
DataValue::I128(i) => dst[..16].copy_from_slice(&i.to_ne_bytes()[..]),
DataValue::F32(f) => dst[..4].copy_from_slice(&f.bits().to_ne_bytes()[..]),
DataValue::F64(f) => dst[..8].copy_from_slice(&f.bits().to_ne_bytes()[..]),
- DataValue::V128(v) => dst[..16].copy_from_slice(&v[..]),
+ DataValue::V128(v) => dst[..16].copy_from_slice(&u128::from_le_bytes(*v).to_ne_bytes()),
_ => unimplemented!(),
};
}
@@ -120,7 +120,7 @@ impl DataValue {
DataValue::B(src[..size].iter().any(|&i| i != 0))
}
_ if ty.is_vector() && ty.bytes() == 16 => {
- DataValue::V128(src[..16].try_into().unwrap())
+ DataValue::V128(u128::from_ne_bytes(src[..16].try_into().unwrap()).to_le_bytes())
}
_ => unimplemented!(),
}
diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs
index 77dcc87e9421..52db56de435c 100644
--- a/cranelift/codegen/src/isa/s390x/abi.rs
+++ b/cranelift/codegen/src/isa/s390x/abi.rs
@@ -97,6 +97,10 @@ fn in_flt_reg(ty: Type) -> bool {
}
}
+fn in_vec_reg(ty: Type) -> bool {
+ ty.is_vector() && ty.bits() == 128
+}
+
fn get_intreg_for_arg(idx: usize) -> Option<Reg> {
match idx {
0 => Some(regs::gpr(2)),
@@ -118,6 +122,20 @@ fn get_fltreg_for_arg(idx: usize) -> Option<Reg> {
}
}
+fn get_vecreg_for_arg(idx: usize) -> Option<Reg> {
+ match idx {
+ 0 => Some(regs::vr(24)),
+ 1 => Some(regs::vr(25)),
+ 2 => Some(regs::vr(26)),
+ 3 => Some(regs::vr(27)),
+ 4 => Some(regs::vr(28)),
+ 5 => Some(regs::vr(29)),
+ 6 => Some(regs::vr(30)),
+ 7 => Some(regs::vr(31)),
+ _ => None,
+ }
+}
+
fn get_intreg_for_ret(idx: usize) -> Option<Reg> {
match idx {
0 => Some(regs::gpr(2)),
@@ -140,6 +158,21 @@ fn get_fltreg_for_ret(idx: usize) -> Option<Reg> {
}
}
+fn get_vecreg_for_ret(idx: usize) -> Option<Reg> {
+ match idx {
+ 0 => Some(regs::vr(24)),
+ // ABI extension to support multi-value returns:
+ 1 => Some(regs::vr(25)),
+ 2 => Some(regs::vr(26)),
+ 3 => Some(regs::vr(27)),
+ 4 => Some(regs::vr(28)),
+ 5 => Some(regs::vr(29)),
+ 6 => Some(regs::vr(30)),
+ 7 => Some(regs::vr(31)),
+ _ => None,
+ }
+}
+
/// This is the limit for the size of argument and return-value areas on the
/// stack. We place a reasonable limit here to avoid integer overflow issues
/// with 32-bit arithmetic: for now, 128 MB.
@@ -182,6 +215,7 @@ impl ABIMachineSpec for S390xMachineDeps {
) -> CodegenResult<(Vec<ABIArg>, i64, Option<usize>)> {
let mut next_gpr = 0;
let mut next_fpr = 0;
+ let mut next_vr = 0;
let mut next_stack: u64 = 0;
let mut ret = vec![];
@@ -206,8 +240,8 @@ impl ABIMachineSpec for S390xMachineDeps {
let intreg = in_int_reg(param.value_type);
let fltreg = in_flt_reg(param.value_type);
- debug_assert!(intreg || fltreg);
- debug_assert!(!(intreg && fltreg));
+ let vecreg = in_vec_reg(param.value_type);
+ debug_assert!(intreg as i32 + fltreg as i32 + vecreg as i32 == 1);
let (next_reg, candidate) = if intreg {
let candidate = match args_or_rets {
@@ -215,12 +249,18 @@ impl ABIMachineSpec for S390xMachineDeps {
ArgsOrRets::Rets => get_intreg_for_ret(next_gpr),
};
(&mut next_gpr, candidate)
- } else {
+ } else if fltreg {
let candidate = match args_or_rets {
ArgsOrRets::Args => get_fltreg_for_arg(next_fpr),
ArgsOrRets::Rets => get_fltreg_for_ret(next_fpr),
};
(&mut next_fpr, candidate)
+ } else {
+ let candidate = match args_or_rets {
+ ArgsOrRets::Args => get_vecreg_for_arg(next_vr),
+ ArgsOrRets::Rets => get_vecreg_for_ret(next_vr),
+ };
+ (&mut next_vr, candidate)
};
// In the Wasmtime ABI only the first return value can be in a register.
@@ -252,7 +292,8 @@ impl ABIMachineSpec for S390xMachineDeps {
// Align the stack slot.
debug_assert!(slot_size.is_power_of_two());
- next_stack = align_to(next_stack, slot_size);
+ let slot_align = std::cmp::min(slot_size, 8);
+ next_stack = align_to(next_stack, slot_align);
// If the type is actually of smaller size (and the argument
// was not extended), it is passed right-aligned.
@@ -477,6 +518,13 @@ impl ABIMachineSpec for S390xMachineDeps {
RegClass::Float => clobbered_fpr.push(reg),
}
}
+ // We need to save the link register in non-leaf functions.
+ // FIXME: This should be included in the clobber list to begin with,
+ // but isn't because we have have excluded call instructions via the
+ // is_included_in_clobbers callback.
+ if outgoing_args_size > 0 {
+ clobbered_gpr.push(Writable::from_reg(RealReg::from(gpr_preg(14))));
+ }
let mut first_clobbered_gpr = 16;
for reg in clobbered_gpr {
@@ -534,13 +582,15 @@ impl ABIMachineSpec for S390xMachineDeps {
// Save FPRs.
for (i, reg) in clobbered_fpr.iter().enumerate() {
- insts.push(Inst::FpuStore64 {
+ insts.push(Inst::VecStoreLane {
+ size: 64,
rd: reg.to_reg().into(),
mem: MemArg::reg_plus_off(
stack_reg(),
(i * 8) as i64 + outgoing_args_size as i64 + fixed_frame_storage_size as i64,
MemFlags::trusted(),
),
+ lane_imm: 0,
});
if flags.unwind_info() {
insts.push(Inst::Unwind {
@@ -566,7 +616,14 @@ impl ABIMachineSpec for S390xMachineDeps {
let mut insts = SmallVec::new();
// Collect clobbered registers.
- let (clobbered_gpr, clobbered_fpr) = get_regs_saved_in_prologue(call_conv, clobbers);
+ let (mut clobbered_gpr, clobbered_fpr) = get_regs_saved_in_prologue(call_conv, clobbers);
+ // We need to restore the link register in non-leaf functions.
+ // FIXME: This should be included in the clobber list to begin with,
+ // but isn't because we have have excluded call instructions via the
+ // is_included_in_clobbers callback.
+ if outgoing_args_size > 0 {
+ clobbered_gpr.push(Writable::from_reg(RealReg::from(gpr_preg(14))));
+ }
let mut first_clobbered_gpr = 16;
for reg in clobbered_gpr {
let enc = reg.to_reg().hw_enc();
@@ -578,13 +635,15 @@ impl ABIMachineSpec for S390xMachineDeps {
// Restore FPRs.
for (i, reg) in clobbered_fpr.iter().enumerate() {
- insts.push(Inst::FpuLoad64 {
+ insts.push(Inst::VecLoadLaneUndef {
+ size: 64,
rd: Writable::from_reg(reg.to_reg().into()),
mem: MemArg::reg_plus_off(
stack_reg(),
(i * 8) as i64 + outgoing_args_size as i64 + fixed_frame_storage_size as i64,
MemFlags::trusted(),
),
+ lane_imm: 0,
});
}
@@ -639,7 +698,7 @@ impl ABIMachineSpec for S390xMachineDeps {
// We allocate in terms of 8-byte slots.
match rc {
RegClass::Int => 1,
- RegClass::Float => 1,
+ RegClass::Float => 2,
}
}
@@ -739,6 +798,21 @@ const fn clobbers() -> PRegSet {
.with(gpr_preg(3))
.with(gpr_preg(4))
.with(gpr_preg(5))
+ // v0 - v7 inclusive and v16 - v31 inclusive are
+ // caller-saves. The upper 64 bits of v8 - v15 inclusive are
+ // also caller-saves. However, because we cannot currently
+ // represent partial registers to regalloc2, we indicate here
+ // that every vector register is caller-save. Because this
+ // function is used at *callsites*, approximating in this
+ // direction (save more than necessary) is conservative and
+ // thus safe.
+ //
+ // Note that we exclude clobbers from a call instruction when
+ // a call instruction's callee has the same ABI as the caller
+ // (the current function body); this is safe (anything
+ // clobbered by callee can be clobbered by caller as well) and
+ // avoids unnecessary saves of v8-v15 in the prologue even
+ // though we include them as defs here.
.with(vr_preg(0))
.with(vr_preg(1))
.with(vr_preg(2))
@@ -747,6 +821,14 @@ const fn clobbers() -> PRegSet {
.with(vr_preg(5))
.with(vr_preg(6))
.with(vr_preg(7))
+ .with(vr_preg(8))
+ .with(vr_preg(9))
+ .with(vr_preg(10))
+ .with(vr_preg(11))
+ .with(vr_preg(12))
+ .with(vr_preg(13))
+ .with(vr_preg(14))
+ .with(vr_preg(15))
.with(vr_preg(16))
.with(vr_preg(17))
.with(vr_preg(18))
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index 00868224c544..38c90e4711da 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -467,26 +467,6 @@
(cond Cond)
(rm Reg))
- ;; A 32-bit move instruction from GPR to FPR or vector element.
- (MovToFpr32
- (rd WritableReg)
- (rn Reg))
-
- ;; A 64-bit move instruction from GPR to FPR or vector element.
- (MovToFpr64
- (rd WritableReg)
- (rn Reg))
-
- ;; A 32-bit move instruction from FPR or vector element to GPR.
- (MovFromFpr32
- (rd WritableReg)
- (rn Reg))
-
- ;; A 64-bit move instruction from FPR or vector element to GPR.
- (MovFromFpr64
- (rd WritableReg)
- (rn Reg))
-
;; 1-op FPU instruction implemented as vector instruction with the W bit.
(FpuRR
(fpu_op FPUOp1)
@@ -508,6 +488,13 @@
(rm Reg)
(ra Reg))
+ ;; 1-op FPU instruction with rounding mode.
+ (FpuRound
+ (op FpuRoundOp)
+ (mode FpuRoundMode)
+ (rd WritableReg)
+ (rn Reg))
+
;; FPU comparison, single-precision (32 bit).
(FpuCmp32
(rn Reg)
@@ -518,69 +505,255 @@
(rn Reg)
(rm Reg))
- ;; Floating-point load, single-precision (32 bit).
- (FpuLoad32
+ ;; Load floating-point constant, single-precision (32 bit).
+ (LoadFpuConst32
(rd WritableReg)
- (mem MemArg))
+ (const_data u32))
- ;; Floating-point store, single-precision (32 bit).
- (FpuStore32
- (rd Reg)
- (mem MemArg))
+ ;; Load floating-point constant, double-precision (64 bit).
+ (LoadFpuConst64
+ (rd WritableReg)
+ (const_data u64))
- ;; Floating-point load, double-precision (64 bit).
- (FpuLoad64
+ ;; A binary vector operation with two vector register sources.
+ (VecRRR
+ (op VecBinaryOp)
(rd WritableReg)
- (mem MemArg))
+ (rn Reg)
+ (rm Reg))
- ;; Floating-point store, double-precision (64 bit).
- (FpuStore64
- (rd Reg)
+ ;; A unary vector operation with a vector register source.
+ (VecRR
+ (op VecUnaryOp)
+ (rd WritableReg)
+ (rn Reg))
+
+ ;; Vector shift instruction with a register source, a register destination,
+ ;; and an immediate plus an optional register as shift count.
+ (VecShiftRR
+ (shift_op VecShiftOp)
+ (rd WritableReg)
+ (rn Reg)
+ (shift_imm u8)
+ (shift_reg Reg))
+
+ ;; Vector select instruction.
+ (VecSelect
+ (rd WritableReg)
+ (rn Reg)
+ (rm Reg)
+ (ra Reg))
+
+ ;; Vector permute instruction.
+ (VecPermute
+ (rd WritableReg)
+ (rn Reg)
+ (rm Reg)
+ (ra Reg))
+
+ ;; Vector permute doubleword immediate instruction.
+ (VecPermuteDWImm
+ (rd WritableReg)
+ (rn Reg)
+ (rm Reg)
+ (idx1 u8)
+ (idx2 u8))
+
+ ;; Vector integer comparison with two register sources and a register
+ ;; destination.
+ (VecIntCmp
+ (op VecIntCmpOp)
+ (rd WritableReg)
+ (rn Reg)
+ (rm Reg))
+
+ ;; Same, but also set the condition code.
+ (VecIntCmpS
+ (op VecIntCmpOp)
+ (rd WritableReg)
+ (rn Reg)
+ (rm Reg))
+
+ ;; Vector floating-point comparison with two register sources and a register
+ ;; destination.
+ (VecFloatCmp
+ (op VecFloatCmpOp)
+ (rd WritableReg)
+ (rn Reg)
+ (rm Reg))
+
+ ;; Same, but also set the condition code.
+ (VecFloatCmpS
+ (op VecFloatCmpOp)
+ (rd WritableReg)
+ (rn Reg)
+ (rm Reg))
+
+ ;; 128-bit vector load instruction.
+ (VecLoad
+ (rd WritableReg)
(mem MemArg))
- ;; Floating-point byte-reversed load, single-precision (32 bit).
- (FpuLoadRev32
+ ;; 128-bit byte-reversed vector load instruction.
+ (VecLoadRev
(rd WritableReg)
(mem MemArg))
- ;; Floating-point byte-reversed store, single-precision (32 bit).
- (FpuStoreRev32
+ ;; 128-bit vector store instruction.
+ (VecStore
+ (rd Reg)
+ (mem MemArg))
+
+ ;; 128-bit byte-reversed vector store instruction.
+ (VecStoreRev
(rd Reg)
(mem MemArg))
- ;; Floating-point byte-reversed load, double-precision (64 bit).
- (FpuLoadRev64
+ ;; 128-bit vector load replicated element instruction.
+ (VecLoadReplicate
+ (size u32)
(rd WritableReg)
(mem MemArg))
- ;; Floating-point byte-reversed store, double-precision (64 bit).
- (FpuStoreRev64
- (rd Reg)
+ ;; 128-bit byte-reversed vector load replicated element instruction.
+ (VecLoadReplicateRev
+ (size u32)
+ (rd WritableReg)
(mem MemArg))
- ;; Load floating-point constant, single-precision (32 bit).
- (LoadFpuConst32
+ ;; Vector move instruction.
+ (VecMov
(rd WritableReg)
- (const_data u32))
+ (rn Reg))
- ;; Load floating-point constant, double-precision (64 bit).
- (LoadFpuConst64
+ ;; Conditional vector move instruction.
+ (VecCMov
+ (rd WritableReg)
+ (cond Cond)
+ (rm Reg))
+
+ ;; A 128-bit move instruction from two GPRs to a VR.
+ (MovToVec128
+ (rd WritableReg)
+ (rn Reg)
+ (rm Reg))
+
+ ;; Load 128-bit (big-endian) vector constant.
+ (VecLoadConst
+ (rd WritableReg)
+ (const_data u128))
+
+ ;; Load 128-bit (big-endian) replicated vector constant.
+ (VecLoadConstReplicate
+ (size u32)
(rd WritableReg)
(const_data u64))
- ;; 1-op FPU instruction with rounding mode.
- (FpuRound
- (op FpuRoundOp)
- (mode FpuRoundMode)
+ ;; Load vector immediate generated via byte mask.
+ (VecImmByteMask
(rd WritableReg)
- (rn Reg))
+ (mask u16))
- ;; Vector select instruction.
- (VecSelect
+ ;; Load vector replicated contiguous bit mask.
+ (VecImmBitMask
+ (size u32)
+ (rd WritableReg)
+ (start_bit u8)
+ (end_bit u8))
+
+ ;; Load vector replicated immediate.
+ (VecImmReplicate
+ (size u32)
+ (rd WritableReg)
+ (imm i16))
+
+ ;; Vector lane insertion with an in/out VR, a memory source,
+ ;; and an immediate as lane index.
+ (VecLoadLane
+ (size u32)
+ (rd WritableReg)
+ (mem MemArg)
+ (lane_imm u8))
+
+ ;; Same as VecLoadLane, but allow undefined input VR.
+ (VecLoadLaneUndef
+ (size u32)
+ (rd WritableReg)
+ (mem MemArg)
+ (lane_imm u8))
+
+ ;; Byte-reversed vector lane insertion with an in/out VR, a memory source,
+ ;; and an immediate as lane index.
+ (VecLoadLaneRev
+ (size u32)
+ (rd WritableReg)
+ (mem MemArg)
+ (lane_imm u8))
+
+ ;; Same as VecLoadLaneRev, but allow undefined input VR.
+ (VecLoadLaneRevUndef
+ (size u32)
+ (rd WritableReg)
+ (mem MemArg)
+ (lane_imm u8))
+
+ ;; Vector lane extraction with a memory destination, a VR source,
+ ;; and an immediate as lane index.
+ (VecStoreLane
+ (size u32)
+ (rd Reg)
+ (mem MemArg)
+ (lane_imm u8))
+
+ ;; Byte-reversed vector lane extraction with a memory destination, a VR source,
+ ;; and an immediate as lane index.
+ (VecStoreLaneRev
+ (size u32)
+ (rd Reg)
+ (mem MemArg)
+ (lane_imm u8))
+
+ ;; Vector lane insertion with an in/out VR, a GPR source,
+ ;; and an immediate plus an optional register as lane index.
+ (VecInsertLane
+ (size u32)
(rd WritableReg)
(rn Reg)
- (rm Reg)
- (ra Reg))
+ (lane_imm u8)
+ (lane_reg Reg))
+
+ ;; Same as VecInsertLane, but allow undefined input VR.
+ (VecInsertLaneUndef
+ (size u32)
+ (rd WritableReg)
+ (rn Reg)
+ (lane_imm u8)
+ (lane_reg Reg))
+
+ ;; Vector lane extraction with a VR source, a GPR destination,
+ ;; and an immediate plus an optional register as lane index.
+ (VecExtractLane
+ (size u32)
+ (rd WritableReg)
+ (rn Reg)
+ (lane_imm u8)
+ (lane_reg Reg))
+
+ ;; Vector lane insertion with an in/out VR, an immediate source,
+ ;; and an immediate as lane index.
+ (VecInsertLaneImm
+ (size u32)
+ (rd WritableReg)
+ (imm i16)
+ (lane_imm u8))
+
+ ;; Vector lane replication with a VR source, a VR destination,
+ ;; and an immediate as lane index.
+ (VecReplicateLane
+ (size u32)
+ (rd WritableReg)
+ (rn Reg)
+ (lane_imm u8))
;; A machine call instruction.
(Call
@@ -807,18 +980,208 @@
(CmpL64Ext32)
))
+;; A binary vector operation.
+(type VecBinaryOp
+ (enum
+ ;; Addition and subtraction
+ (Add8x16)
+ (Add16x8)
+ (Add32x4)
+ (Add64x2)
+ (Sub8x16)
+ (Sub16x8)
+ (Sub32x4)
+ (Sub64x2)
+ ;; Multiplication (64-bit not supported)
+ (Mul8x16)
+ (Mul16x8)
+ (Mul32x4)
+ (UMulHi8x16)
+ (UMulHi16x8)
+ (UMulHi32x4)
+ (SMulHi8x16)
+ (SMulHi16x8)
+ (SMulHi32x4)
+ (UMulEven8x16)
+ (UMulEven16x8)
+ (UMulEven32x4)
+ (SMulEven8x16)
+ (SMulEven16x8)
+ (SMulEven32x4)
+ (UMulOdd8x16)
+ (UMulOdd16x8)
+ (UMulOdd32x4)
+ (SMulOdd8x16)
+ (SMulOdd16x8)
+ (SMulOdd32x4)
+ ;; Minimum, maximum, and average
+ (UMax8x16)
+ (UMax16x8)
+ (UMax32x4)
+ (UMax64x2)
+ (SMax8x16)
+ (SMax16x8)
+ (SMax32x4)
+ (SMax64x2)
+ (UMin8x16)
+ (UMin16x8)
+ (UMin32x4)
+ (UMin64x2)
+ (SMin8x16)
+ (SMin16x8)
+ (SMin32x4)
+ (SMin64x2)
+ (UAvg8x16)
+ (UAvg16x8)
+ (UAvg32x4)
+ (UAvg64x2)
+ (SAvg8x16)
+ (SAvg16x8)
+ (SAvg32x4)
+ (SAvg64x2)
+ ;; Bitwise operations
+ (And128)
+ (Orr128)
+ (Xor128)
+ (NotAnd128)
+ (NotOrr128)
+ (NotXor128)
+ (AndNot128)
+ (OrrNot128)
+ ;; Bit permute
+ (BitPermute128)
+ ;; Full vector shift operations
+ (LShLByByte128)
+ (LShRByByte128)
+ (AShRByByte128)
+ (LShLByBit128)
+ (LShRByBit128)
+ (AShRByBit128)
+ ;; Pack
+ (Pack16x8)
+ (Pack32x4)
+ (Pack64x2)
+ ;; Pack saturate (unsigned)
+ (PackUSat16x8)
+ (PackUSat32x4)
+ (PackUSat64x2)
+ ;; Pack saturate (signed)
+ (PackSSat16x8)
+ (PackSSat32x4)
+ (PackSSat64x2)
+ ;; Merge
+ (MergeLow8x16)
+ (MergeLow16x8)
+ (MergeLow32x4)
+ (MergeLow64x2)
+ (MergeHigh8x16)
+ (MergeHigh16x8)
+ (MergeHigh32x4)
+ (MergeHigh64x2)
+))
+
+;; A vector unary operation.
+(type VecUnaryOp
+ (enum
+ ;; Sign operations
+ (Abs8x16)
+ (Abs16x8)
+ (Abs32x4)
+ (Abs64x2)
+ (Neg8x16)
+ (Neg16x8)
+ (Neg32x4)
+ (Neg64x2)
+ ;; Population count
+ (Popcnt8x16)
+ (Popcnt16x8)
+ (Popcnt32x4)
+ (Popcnt64x2)
+ ;; Unpack
+ (UnpackULow8x16)
+ (UnpackULow16x8)
+ (UnpackULow32x4)
+ (UnpackUHigh8x16)
+ (UnpackUHigh16x8)
+ (UnpackUHigh32x4)
+ (UnpackSLow8x16)
+ (UnpackSLow16x8)
+ (UnpackSLow32x4)
+ (UnpackSHigh8x16)
+ (UnpackSHigh16x8)
+ (UnpackSHigh32x4)
+))
+
+;; A vector shift operation.
+(type VecShiftOp
+ (enum
+ (RotL8x16)
+ (RotL16x8)
+ (RotL32x4)
+ (RotL64x2)
+ (LShL8x16)
+ (LShL16x8)
+ (LShL32x4)
+ (LShL64x2)
+ (LShR8x16)
+ (LShR16x8)
+ (LShR32x4)
+ (LShR64x2)
+ (AShR8x16)
+ (AShR16x8)
+ (AShR32x4)
+ (AShR64x2)
+))
+
+;; An integer vector comparison operation.
+(type VecIntCmpOp
+ (enum
+ (CmpEq8x16)
+ (CmpEq16x8)
+ (CmpEq32x4)
+ (CmpEq64x2)
+ (SCmpHi8x16)
+ (SCmpHi16x8)
+ (SCmpHi32x4)
+ (SCmpHi64x2)
+ (UCmpHi8x16)
+ (UCmpHi16x8)
+ (UCmpHi32x4)
+ (UCmpHi64x2)
+))
+
+;; A floatint-point vector comparison operation.
+(type VecFloatCmpOp
+ (enum
+ (CmpEq32x4)
+ (CmpEq64x2)
+ (CmpHi32x4)
+ (CmpHi64x2)
+ (CmpHiEq32x4)
+ (CmpHiEq64x2)
+))
+
;; A floating-point unit (FPU) operation with one arg.
(type FPUOp1
(enum
(Abs32)
(Abs64)
+ (Abs32x4)
+ (Abs64x2)
(Neg32)
(Neg64)
+ (Neg32x4)
+ (Neg64x2)
(NegAbs32)
(NegAbs64)
+ (NegAbs32x4)
+ (NegAbs64x2)
(Sqrt32)
(Sqrt64)
+ (Sqrt32x4)
+ (Sqrt64x2)
(Cvt32To64)
+ (Cvt32x4To64x2)
))
;; A floating-point unit (FPU) operation with two args.
@@ -826,16 +1189,36 @@
(enum
(Add32)
(Add64)
+ (Add32x4)
+ (Add64x2)
(Sub32)
(Sub64)
+ (Sub32x4)
+ (Sub64x2)
(Mul32)
(Mul64)
+ (Mul32x4)
+ (Mul64x2)
(Div32)
(Div64)
+ (Div32x4)
+ (Div64x2)
(Max32)
(Max64)
+ (Max32x4)
+ (Max64x2)
(Min32)
(Min64)
+ (Min32x4)
+ (Min64x2)
+ (MaxPseudo32)
+ (MaxPseudo64)
+ (MaxPseudo32x4)
+ (MaxPseudo64x2)
+ (MinPseudo32)
+ (MinPseudo64)
+ (MinPseudo32x4)
+ (MinPseudo64x2)
))
;; A floating-point unit (FPU) operation with three args.
@@ -843,24 +1226,39 @@
(enum
(MAdd32)
(MAdd64)
+ (MAdd32x4)
+ (MAdd64x2)
(MSub32)
(MSub64)
+ (MSub32x4)
+ (MSub64x2)
))
;; A floating-point unit (FPU) operation with one arg, and rounding mode.
(type FpuRoundOp
(enum
(Cvt64To32)
+ (Cvt64x2To32x4)
(Round32)
(Round64)
+ (Round32x4)
+ (Round64x2)
(ToSInt32)
(ToSInt64)
(ToUInt32)
(ToUInt64)
+ (ToSInt32x4)
+ (ToSInt64x2)
+ (ToUInt32x4)
+ (ToUInt64x2)
(FromSInt32)
(FromSInt64)
(FromUInt32)
(FromUInt64)
+ (FromSInt32x4)
+ (FromSInt64x2)
+ (FromUInt32x4)
+ (FromUInt64x2)
))
;; Rounding modes for floating-point ops.
@@ -949,6 +1347,55 @@
(decl u64_as_i16 (u64) i16)
(extern constructor u64_as_i16 u64_as_i16)
+;; Construct and extract immediate vector constants.
+
+(decl u64_pair (u64 u64) u128)
+(extern constructor u64_pair u64_pair_concat)
+(extern extractor infallible u64_pair u64_pair_split)
+
+(decl u32_pair (u32 u32) u64)
+(extern constructor u32_pair u32_pair_concat)
+(extern extractor infallible u32_pair u32_pair_split)
+
+(decl u16_pair (u16 u16) u32)
+(extern constructor u16_pair u16_pair_concat)
+(extern extractor infallible u16_pair u16_pair_split)
+
+(decl u8_pair (u8 u8) u16)
+(extern constructor u8_pair u8_pair_concat)
+(extern extractor infallible u8_pair u8_pair_split)
+
+(decl imm8x16 (u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8 u8) u128)
+(extractor (imm8x16 a b c d e f g h i j k l m n o p)
+ (u64_pair (u32_pair (u16_pair (u8_pair a b) (u8_pair c d))
+ (u16_pair (u8_pair e f) (u8_pair g h)))
+ (u32_pair (u16_pair (u8_pair i j) (u8_pair k l))
+ (u16_pair (u8_pair m n) (u8_pair o p)))))
+(rule (imm8x16 a b c d e f g h i j k l m n o p)
+ (u64_pair (u32_pair (u16_pair (u8_pair a b) (u8_pair c d))
+ (u16_pair (u8_pair e f) (u8_pair g h)))
+ (u32_pair (u16_pair (u8_pair i j) (u8_pair k l))
+ (u16_pair (u8_pair m n) (u8_pair o p)))))
+
+;; Convert a little-endian lane index to a big-endian lane index.
+
+(decl be_lane_idx (Type u8) u8)
+(extern constructor be_lane_idx be_lane_idx)
+
+;; Construct a VGBM mask to set all bits in one lane of a vector.
+
+(decl lane_byte_mask (Type u8) u16)
+(extern constructor lane_byte_mask lane_byte_mask)
+
+;; Extract "permute" and "and" masks from a shuffle constant
+
+(decl shuffle_mask_from_u128 (u128 u16) u128)
+(extern extractor infallible shuffle_mask_from_u128 shuffle_mask_from_u128)
+
+(decl shuffle_mask (u128 u16) Immediate)
+(extractor (shuffle_mask permute_mask and_mask)
+ (u128_from_immediate (shuffle_mask_from_u128 permute_mask and_mask)))
+
;; Split an u64 into high and low parts.
(decl u64_nonzero_hipart (u64) u64)
@@ -965,6 +1412,9 @@
(decl i16_from_u64 (i16) u64)
(extern extractor i16_from_u64 i16_from_u64)
+(decl i16_from_u32 (i16) u32)
+(extern extractor i16_from_u32 i16_from_u32)
+
(decl uimm32shifted_from_u64 (UImm32Shifted) u64)
(extern extractor uimm32shifted_from_u64 uimm32shifted_from_u64)
@@ -985,6 +1435,9 @@
(decl u64_from_signed_value (u64) Value)
(extern extractor u64_from_signed_value u64_from_signed_value)
+(decl u64_from_inverted_value (u64) Value)
+(extern extractor u64_from_inverted_value u64_from_inverted_value)
+
(decl i64_from_value (i64) Value)
(extern extractor i64_from_value i64_from_value)
@@ -1097,10 +1550,10 @@
(type MemArg extern (enum))
-(decl memarg_reg_plus_reg (Reg Reg MemFlags) MemArg)
+(decl memarg_reg_plus_reg (Reg Reg u8 MemFlags) MemArg)
(extern constructor memarg_reg_plus_reg memarg_reg_plus_reg)
-(decl memarg_reg_plus_off (Reg i64 MemFlags) MemArg)
+(decl memarg_reg_plus_off (Reg i64 u8 MemFlags) MemArg)
(extern constructor memarg_reg_plus_off memarg_reg_plus_off)
(decl memarg_symbol (ExternalName i32 MemFlags) MemArg)
@@ -1126,10 +1579,10 @@
(decl lower_address (MemFlags Value Offset32) MemArg)
(rule (lower_address flags addr (i64_from_offset offset))
- (memarg_reg_plus_off addr offset flags))
+ (memarg_reg_plus_off addr offset 0 flags))
(rule (lower_address flags (iadd x y) (i64_from_offset 0))
- (memarg_reg_plus_reg x y flags))
+ (memarg_reg_plus_reg x y 0 flags))
(rule (lower_address flags
(symbol_value (symbol_value_data name (reloc_distance_near) sym_offset))
@@ -1138,6 +1591,17 @@
(memarg_symbol name final_offset flags))
+;; Lower an address plus a small bias into a `MemArg`.
+
+(decl lower_address_bias (MemFlags Value Offset32 u8) MemArg)
+
+(rule (lower_address_bias flags addr (i64_from_offset offset) bias)
+ (memarg_reg_plus_off addr offset bias flags))
+
+(rule (lower_address_bias flags (iadd x y) (i64_from_offset 0) bias)
+ (memarg_reg_plus_reg x y bias flags))
+
+
;; Test whether a `load` address will be lowered to a `MemArg::Symbol`.
(decl pure load_sym (Inst) Inst)
@@ -1206,6 +1670,11 @@
(extractor (sinkable_load_16 inst)
(and (value_type $I16) (sinkable_load inst)))
+;; Sinkable little-endian load instruction.
+(decl sinkable_load_little (Inst) Value)
+(extractor (sinkable_load_little inst)
+ (sinkable_inst (and inst (load (littleendian) _addr _offset))))
+
;; Sinkable big-endian sload16 instruction.
(decl sinkable_sload16 (Inst) Value)
(extractor (sinkable_sload16 inst)
@@ -1615,87 +2084,225 @@
(_ Unit (emit (MInst.FpuRound op mode dst src))))
dst))
-;; Helper for emitting `MInst.MovToFpr32` instructions.
-(decl mov_to_fpr32 (Reg) Reg)
-(rule (mov_to_fpr32 src)
- (let ((dst WritableReg (temp_writable_reg $F32))
- (_ Unit (emit (MInst.MovToFpr32 dst src))))
+;; Helper for emitting `MInst.VecRRR` instructions.
+(decl vec_rrr (Type VecBinaryOp Reg Reg) Reg)
+(rule (vec_rrr ty op src1 src2)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecRRR op dst src1 src2))))
dst))
-;; Helper for emitting `MInst.MovToFpr64` instructions.
-(decl mov_to_fpr64 (Reg) Reg)
-(rule (mov_to_fpr64 src)
- (let ((dst WritableReg (temp_writable_reg $F64))
- (_ Unit (emit (MInst.MovToFpr64 dst src))))
+;; Helper for emitting `MInst.VecRR` instructions.
+(decl vec_rr (Type VecUnaryOp Reg) Reg)
+(rule (vec_rr ty op src)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecRR op dst src))))
dst))
-;; Helper for emitting `MInst.MovFromFpr32` instructions.
-(decl mov_from_fpr32 (Reg) Reg)
-(rule (mov_from_fpr32 src)
- (let ((dst WritableReg (temp_writable_reg $I32))
- (_ Unit (emit (MInst.MovFromFpr32 dst src))))
+;; Helper for emitting `MInst.VecShiftRR` instructions.
+(decl vec_shift_rr (Type VecShiftOp Reg u8 Reg) Reg)
+(rule (vec_shift_rr ty op src shift_imm shift_reg)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecShiftRR op dst src shift_imm shift_reg))))
dst))
-;; Helper for emitting `MInst.MovFromFpr64` instructions.
-(decl mov_from_fpr64 (Reg) Reg)
-(rule (mov_from_fpr64 src)
- (let ((dst WritableReg (temp_writable_reg $I64))
- (_ Unit (emit (MInst.MovFromFpr64 dst src))))
+;; Helper for emitting `MInst.VecSelect` instructions.
+(decl vec_select (Type Reg Reg Reg) Reg)
+(rule (vec_select ty src1 src2 src3)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecSelect dst src1 src2 src3))))
+ dst))
+
+;; Helper for emitting `MInst.VecPermute` instructions.
+(decl vec_permute (Type Reg Reg Reg) Reg)
+(rule (vec_permute ty src1 src2 src3)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecPermute dst src1 src2 src3))))
+ dst))
+
+;; Helper for emitting `MInst.VecPermuteDWImm` instructions.
+(decl vec_permute_dw_imm (Type Reg u8 Reg u8) Reg)
+(rule (vec_permute_dw_imm ty src1 idx1 src2 idx2)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecPermuteDWImm dst src1 src2 idx1 idx2))))
dst))
-;; Helper for emitting `MInst.FpuLoad32` instructions.
-(decl fpu_load32 (MemArg) Reg)
-(rule (fpu_load32 addr)
- (let ((dst WritableReg (temp_writable_reg $F32))
- (_ Unit (emit (MInst.FpuLoad32 dst addr))))
+;; Helper for emitting `MInst.VecIntCmp` instructions.
+(decl vec_int_cmp (Type VecIntCmpOp Reg Reg) Reg)
+(rule (vec_int_cmp ty op src1 src2)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecIntCmp op dst src1 src2))))
dst))
-;; Helper for emitting `MInst.FpuLoad64` instructions.
-(decl fpu_load64 (MemArg) Reg)
-(rule (fpu_load64 addr)
- (let ((dst WritableReg (temp_writable_reg $F64))
- (_ Unit (emit (MInst.FpuLoad64 dst addr))))
+;; Helper for emitting `MInst.VecIntCmpS` instructions.
+(decl vec_int_cmps (Type VecIntCmpOp Reg Reg) ProducesFlags)
+(rule (vec_int_cmps ty op src1 src2)
+ (let ((tmp WritableReg (temp_writable_reg ty)))
+ (ProducesFlags.ProducesFlagsSideEffect (MInst.VecIntCmpS op tmp src1 src2))))
+
+;; Helper for emitting `MInst.VecFloatCmp` instructions.
+(decl vec_float_cmp (Type VecFloatCmpOp Reg Reg) Reg)
+(rule (vec_float_cmp ty op src1 src2)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecFloatCmp op dst src1 src2))))
dst))
-;; Helper for emitting `MInst.FpuLoadRev32` instructions.
-(decl fpu_loadrev32 (MemArg) Reg)
-(rule (fpu_loadrev32 addr)
- (let ((dst WritableReg (temp_writable_reg $F32))
- (_ Unit (emit (MInst.FpuLoadRev32 dst addr))))
+;; Helper for emitting `MInst.VecFloatCmpS` instructions.
+(decl vec_float_cmps (Type VecFloatCmpOp Reg Reg) ProducesFlags)
+(rule (vec_float_cmps ty op src1 src2)
+ (let ((tmp WritableReg (temp_writable_reg ty)))
+ (ProducesFlags.ProducesFlagsSideEffect (MInst.VecFloatCmpS op tmp src1 src2))))
+
+;; Helper for emitting `MInst.VecLoad` instructions.
+(decl vec_load (Type MemArg) Reg)
+(rule (vec_load ty addr)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecLoad dst addr))))
dst))
-;; Helper for emitting `MInst.FpuLoadRev64` instructions.
-(decl fpu_loadrev64 (MemArg) Reg)
-(rule (fpu_loadrev64 addr)
- (let ((dst WritableReg (temp_writable_reg $F64))
- (_ Unit (emit (MInst.FpuLoadRev64 dst addr))))
+;; Helper for emitting `MInst.VecLoadRev` instructions.
+(decl vec_loadrev (Type MemArg) Reg)
+(rule (vec_loadrev ty addr)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecLoadRev dst addr))))
dst))
-;; Helper for emitting `MInst.FpuStore32` instructions.
-(decl fpu_store32 (Reg MemArg) SideEffectNoResult)
-(rule (fpu_store32 src addr)
- (SideEffectNoResult.Inst (MInst.FpuStore32 src addr)))
+;; Helper for emitting `MInst.VecStore` instructions.
+(decl vec_store (Reg MemArg) SideEffectNoResult)
+(rule (vec_store src addr)
+ (SideEffectNoResult.Inst (MInst.VecStore src addr)))
-;; Helper for emitting `MInst.FpuStore64` instructions.
-(decl fpu_store64 (Reg MemArg) SideEffectNoResult)
-(rule (fpu_store64 src addr)
- (SideEffectNoResult.Inst (MInst.FpuStore64 src addr)))
+;; Helper for emitting `MInst.VecStoreRev` instructions.
+(decl vec_storerev (Reg MemArg) SideEffectNoResult)
+(rule (vec_storerev src addr)
+ (SideEffectNoResult.Inst (MInst.VecStoreRev src addr)))
-;; Helper for emitting `MInst.FpuStoreRev32` instructions.
-(decl fpu_storerev32 (Reg MemArg) SideEffectNoResult)
-(rule (fpu_storerev32 src addr)
- (SideEffectNoResult.Inst (MInst.FpuStoreRev32 src addr)))
+;; Helper for emitting `MInst.VecLoadReplicate` instructions.
+(decl vec_load_replicate (Type MemArg) Reg)
+(rule (vec_load_replicate (ty_vec128 ty @ (multi_lane size _)) addr)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecLoadReplicate size dst addr))))
+ dst))
-;; Helper for emitting `MInst.FpuStoreRev64` instructions.
-(decl fpu_storerev64 (Reg MemArg) SideEffectNoResult)
-(rule (fpu_storerev64 src addr)
- (SideEffectNoResult.Inst (MInst.FpuStoreRev64 src addr)))
+;; Helper for emitting `MInst.VecLoadReplicateRev` instructions.
+(decl vec_load_replicate_rev (Type MemArg) Reg)
+(rule (vec_load_replicate_rev (ty_vec128 ty @ (multi_lane size _)) addr)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecLoadReplicateRev size dst addr))))
+ dst))
-;; Helper for emitting `MInst.VecSelect` instructions.
-(decl vec_select (Type Reg Reg Reg) Reg)
-(rule (vec_select ty src1 src2 src3)
+;; Helper for emitting `MInst.MovToVec128` instructions.
+(decl mov_to_vec128 (Type Reg Reg) Reg)
+(rule (mov_to_vec128 ty src1 src2)
(let ((dst WritableReg (temp_writable_reg ty))
- (_ Unit (emit (MInst.VecSelect dst src1 src2 src3))))
+ (_ Unit (emit (MInst.MovToVec128 dst src1 src2))))
+ dst))
+
+;; Helper for emitting `MInst.VecLoadConst` instructions.
+(decl vec_load_const (Type u128) Reg)
+(rule (vec_load_const (ty_vec128 ty) n)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecLoadConst dst n))))
+ dst))
+
+;; Helper for emitting `MInst.VecLoadConstReplicate` instructions.
+(decl vec_load_const_replicate (Type u64) Reg)
+(rule (vec_load_const_replicate ty @ (multi_lane size _) n)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecLoadConstReplicate size dst n))))
+ dst))
+
+;; Helper for emitting `MInst.VecImmByteMask` instructions.
+(decl vec_imm_byte_mask (Type u16) Reg)
+(rule (vec_imm_byte_mask (ty_vec128 ty) n)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecImmByteMask dst n))))
+ dst))
+
+;; Helper for emitting `MInst.VecImmBitMask` instructions.
+(decl vec_imm_bit_mask (Type u8 u8) Reg)
+(rule (vec_imm_bit_mask (ty_vec128 ty @ (multi_lane size _)) start_bit end_bit)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecImmBitMask size dst start_bit end_bit))))
+ dst))
+
+;; Helper for emitting `MInst.VecImmReplicate` instructions.
+(decl vec_imm_replicate (Type i16) Reg)
+(rule (vec_imm_replicate (ty_vec128 ty @ (multi_lane size _)) n)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecImmReplicate size dst n))))
+ dst))
+
+;; Helper for emitting `MInst.VecLoadLane` instructions.
+(decl vec_load_lane (Type Reg MemArg u8) Reg)
+(rule (vec_load_lane ty @ (multi_lane size _) src addr lane_imm)
+ (let ((dst WritableReg (copy_writable_reg ty src))
+ (_ Unit (emit (MInst.VecLoadLane size dst addr lane_imm))))
+ dst))
+
+;; Helper for emitting `MInst.VecLoadLaneUndef` instructions.
+(decl vec_load_lane_undef (Type MemArg u8) Reg)
+(rule (vec_load_lane_undef ty @ (multi_lane size _) addr lane_imm)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecLoadLaneUndef size dst addr lane_imm))))
+ dst))
+
+;; Helper for emitting `MInst.VecLoadLaneRev` instructions.
+(decl vec_load_lane_rev (Type Reg MemArg u8) Reg)
+(rule (vec_load_lane_rev ty @ (multi_lane size _) src addr lane_imm)
+ (let ((dst WritableReg (copy_writable_reg ty src))
+ (_ Unit (emit (MInst.VecLoadLaneRev size dst addr lane_imm))))
+ dst))
+
+;; Helper for emitting `MInst.VecLoadLaneRevUndef` instructions.
+(decl vec_load_lane_rev_undef (Type MemArg u8) Reg)
+(rule (vec_load_lane_rev_undef ty @ (multi_lane size _) addr lane_imm)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecLoadLaneRevUndef size dst addr lane_imm))))
+ dst))
+
+;; Helper for emitting `MInst.VecStoreLane` instructions.
+(decl vec_store_lane (Type Reg MemArg u8) SideEffectNoResult)
+(rule (vec_store_lane ty @ (multi_lane size _) src addr lane_imm)
+ (SideEffectNoResult.Inst (MInst.VecStoreLane size src addr lane_imm)))
+
+;; Helper for emitting `MInst.VecStoreLaneRev` instructions.
+(decl vec_store_lane_rev (Type Reg MemArg u8) SideEffectNoResult)
+(rule (vec_store_lane_rev ty @ (multi_lane size _) src addr lane_imm)
+ (SideEffectNoResult.Inst (MInst.VecStoreLaneRev size src addr lane_imm)))
+
+;; Helper for emitting `MInst.VecInsertLane` instructions.
+(decl vec_insert_lane (Type Reg Reg u8 Reg) Reg)
+(rule (vec_insert_lane ty @ (multi_lane size _) src1 src2 lane_imm lane_reg)
+ (let ((dst WritableReg (copy_writable_reg ty src1))
+ (_ Unit (emit (MInst.VecInsertLane size dst src2 lane_imm lane_reg))))
+ dst))
+
+;; Helper for emitting `MInst.VecInsertLaneUndef` instructions.
+(decl vec_insert_lane_undef (Type Reg u8 Reg) Reg)
+(rule (vec_insert_lane_undef ty @ (multi_lane size _) src lane_imm lane_reg)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecInsertLaneUndef size dst src lane_imm lane_reg))))
+ dst))
+
+;; Helper for emitting `MInst.VecExtractLane` instructions.
+(decl vec_extract_lane (Type Reg u8 Reg) Reg)
+(rule (vec_extract_lane (multi_lane size _) src lane_imm lane_reg)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.VecExtractLane size dst src lane_imm lane_reg))))
+ dst))
+
+;; Helper for emitting `MInst.VecInsertLaneImm` instructions.
+(decl vec_insert_lane_imm (Type Reg i16 u8) Reg)
+(rule (vec_insert_lane_imm ty @ (multi_lane size _) src imm lane_imm)
+ (let ((dst WritableReg (copy_writable_reg ty src))
+ (_ Unit (emit (MInst.VecInsertLaneImm size dst imm lane_imm))))
+ dst))
+
+;; Helper for emitting `MInst.VecReplicateLane` instructions.
+(decl vec_replicate_lane (Type Reg u8) Reg)
+(rule (vec_replicate_lane ty @ (multi_lane size _) src lane_imm)
+ (let ((dst WritableReg (temp_writable_reg ty))
+ (_ Unit (emit (MInst.VecReplicateLane size dst src lane_imm))))
dst))
;; Helper for emitting `MInst.LoadExtNameFar` instructions.
@@ -1858,6 +2465,9 @@
(rule (emit_mov $F64 dst src)
(emit (MInst.FpuMove64 dst src)))
+(rule (emit_mov (ty_vec128 ty) dst src)
+ (emit (MInst.VecMov dst src)))
+
;; Allocate a temporary (writable) register, initialized as a copy of the input.
(decl copy_writable_reg (Type Reg) WritableReg)
(rule (copy_writable_reg ty src)
@@ -1888,8 +2498,12 @@
(rule (emit_arg_store $I32 reg mem) (emit_side_effect (store32 reg mem)))
(rule (emit_arg_store $I64 reg mem) (emit_side_effect (store64 reg mem)))
(rule (emit_arg_store $R64 reg mem) (emit_side_effect (store64 reg mem)))
-(rule (emit_arg_store $F32 reg mem) (emit_side_effect (fpu_store32 reg mem)))
-(rule (emit_arg_store $F64 reg mem) (emit_side_effect (fpu_store64 reg mem)))
+(rule (emit_arg_store $F32 reg mem)
+ (emit_side_effect (vec_store_lane $F32X4 reg mem 0)))
+(rule (emit_arg_store $F64 reg mem)
+ (emit_side_effect (vec_store_lane $F64X2 reg mem 0)))
+(rule (emit_arg_store (ty_vec128 ty) reg mem)
+ (emit_side_effect (vec_store reg mem)))
(decl emit_arg_load (Type MemArg) Reg)
(rule (emit_arg_load $I8 mem) (zext32_mem $I8 mem))
@@ -1897,8 +2511,9 @@
(rule (emit_arg_load $I32 mem) (load32 mem))
(rule (emit_arg_load $I64 mem) (load64 mem))
(rule (emit_arg_load $R64 mem) (load64 mem))
-(rule (emit_arg_load $F32 mem) (fpu_load64 mem))
-(rule (emit_arg_load $F64 mem) (fpu_load64 mem))
+(rule (emit_arg_load $F32 mem) (vec_load_lane_undef $F32X4 mem 0))
+(rule (emit_arg_load $F64 mem) (vec_load_lane_undef $F64X2 mem 0))
+(rule (emit_arg_load (ty_vec128 ty) mem) (vec_load ty mem))
;; Copy a single argument/return value to its slots.
(decl copy_to_arg (i64 ABIArg Value) Unit)
@@ -2026,6 +2641,36 @@
(_ Unit (emit (MInst.Mov64SImm32 dst n))))
(writable_reg_to_reg dst)))
+;; Allocate a temporary register, initialized with a vector immediate.
+(decl vec_imm (Type u128) Reg)
+(rule (vec_imm (ty_vec128 ty) 0)
+ (vec_imm_byte_mask ty 0))
+(rule (vec_imm (ty_vec128 ty) (u64_pair n n))
+ (vec_imm_splat $I64X2 n))
+(rule (vec_imm (ty_vec128 ty) n)
+ (vec_load_const ty n))
+
+;; Variant with replicated immediate.
+(decl vec_imm_splat (Type u64) Reg)
+(rule (vec_imm_splat (ty_vec128 ty) 0)
+ (vec_imm_byte_mask ty 0))
+(rule (vec_imm_splat ty @ (multi_lane 8 _) n)
+ (vec_imm_replicate ty (u64_as_i16 n)))
+(rule (vec_imm_splat ty @ (multi_lane 16 _) n)
+ (vec_imm_replicate ty (u64_as_i16 n)))
+(rule (vec_imm_splat ty @ (multi_lane 32 _) (u32_pair _ (i16_from_u32 n)))
+ (vec_imm_replicate ty n))
+(rule (vec_imm_splat ty @ (multi_lane 64 _) (i16_from_u64 n))
+ (vec_imm_replicate ty n))
+(rule (vec_imm_splat (multi_lane 16 _) (u32_pair _ (u16_pair _ (u8_pair n n))))
+ (vec_imm_splat $I8X16 (u8_as_u64 n)))
+(rule (vec_imm_splat (multi_lane 32 _) (u32_pair _ (u16_pair n n)))
+ (vec_imm_splat $I16X8 (u16_as_u64 n)))
+(rule (vec_imm_splat (multi_lane 64 _) (u32_pair n n))
+ (vec_imm_splat $I32X4 (u32_as_u64 n)))
+(rule (vec_imm_splat (ty_vec128 ty) n)
+ (vec_load_const_replicate ty n))
+
;; Place an immediate into the low half of a register pair.
;; The high half is taken from the input.
(decl imm_regpair_lo (Type u64 RegPair) RegPair)
@@ -2337,6 +2982,10 @@
(rule (emit_cmov_reg $F64 dst cond src)
(ConsumesFlags.ConsumesFlagsReturnsReg (MInst.FpuCMov64 dst cond src)
dst))
+(rule (emit_cmov_reg (ty_vec128 ty) dst cond src)
+ (ConsumesFlags.ConsumesFlagsReturnsReg (MInst.VecCMov dst cond src)
+ dst))
+
;; Conditionally select between two source registers.
(decl cmov_reg (Type Cond Reg Reg) ConsumesFlags)
@@ -2488,7 +3137,7 @@
(decl casloop_emit (VecMInstBuilder Type MemFlags Reg Reg) Reg)
(rule (casloop_emit ib ty flags aligned_addr val)
(let (;; Construct a memory argument for the aligned word.
- (aligned_mem MemArg (memarg_reg_plus_off aligned_addr 0 flags))
+ (aligned_mem MemArg (memarg_reg_plus_off aligned_addr 0 0 flags))
;; Add the compare-and-swap instruction to the builder.
(result Reg (push_atomic_cas ib (ty_ext32 ty)
(casloop_val_reg) val aligned_mem))
@@ -2607,6 +3256,91 @@
(extern constructor abi_accumulate_outgoing_args_size abi_accumulate_outgoing_args_size)
+;; Helpers for generating vector pack and unpack instructions ;;;;;;;;;;;;;;;;;;
+
+(decl vec_widen_type (Type) Type)
+(rule (vec_widen_type $I8X16) $I16X8)
+(rule (vec_widen_type $I16X8) $I32X4)
+(rule (vec_widen_type $I32X4) $I64X2)
+
+(decl vecop_pack (Type) VecBinaryOp)
+(rule (vecop_pack $I16X8) (VecBinaryOp.Pack16x8))
+(rule (vecop_pack $I32X4) (VecBinaryOp.Pack32x4))
+(rule (vecop_pack $I64X2) (VecBinaryOp.Pack64x2))
+
+(decl vec_pack (Type Reg Reg) Reg)
+(rule (vec_pack ty x y) (vec_rrr ty (vecop_pack ty) x y))
+
+(decl vecop_pack_ssat (Type) VecBinaryOp)
+(rule (vecop_pack_ssat $I16X8) (VecBinaryOp.PackSSat16x8))
+(rule (vecop_pack_ssat $I32X4) (VecBinaryOp.PackSSat32x4))
+(rule (vecop_pack_ssat $I64X2) (VecBinaryOp.PackSSat64x2))
+
+(decl vec_pack_ssat (Type Reg Reg) Reg)
+(rule (vec_pack_ssat ty x y) (vec_rrr ty (vecop_pack_ssat ty) x y))
+
+(decl vecop_pack_usat (Type) VecBinaryOp)
+(rule (vecop_pack_usat $I16X8) (VecBinaryOp.PackUSat16x8))
+(rule (vecop_pack_usat $I32X4) (VecBinaryOp.PackUSat32x4))
+(rule (vecop_pack_usat $I64X2) (VecBinaryOp.PackUSat64x2))
+
+(decl vec_pack_usat (Type Reg Reg) Reg)
+(rule (vec_pack_usat ty x y) (vec_rrr ty (vecop_pack_usat ty) x y))
+
+(decl vecop_unpacks_low (Type) VecUnaryOp)
+(rule (vecop_unpacks_low $I8X16) (VecUnaryOp.UnpackSLow8x16))
+(rule (vecop_unpacks_low $I16X8) (VecUnaryOp.UnpackSLow16x8))
+(rule (vecop_unpacks_low $I32X4) (VecUnaryOp.UnpackSLow32x4))
+
+(decl vec_unpacks_low (Type Reg) Reg)
+(rule (vec_unpacks_low ty x) (vec_rr ty (vecop_unpacks_low ty) x))
+
+(decl vecop_unpacks_high (Type) VecUnaryOp)
+(rule (vecop_unpacks_high $I8X16) (VecUnaryOp.UnpackSHigh8x16))
+(rule (vecop_unpacks_high $I16X8) (VecUnaryOp.UnpackSHigh16x8))
+(rule (vecop_unpacks_high $I32X4) (VecUnaryOp.UnpackSHigh32x4))
+
+(decl vec_unpacks_high (Type Reg) Reg)
+(rule (vec_unpacks_high ty x) (vec_rr ty (vecop_unpacks_high ty) x))
+
+(decl vecop_unpacku_low (Type) VecUnaryOp)
+(rule (vecop_unpacku_low $I8X16) (VecUnaryOp.UnpackULow8x16))
+(rule (vecop_unpacku_low $I16X8) (VecUnaryOp.UnpackULow16x8))
+(rule (vecop_unpacku_low $I32X4) (VecUnaryOp.UnpackULow32x4))
+
+(decl vec_unpacku_low (Type Reg) Reg)
+(rule (vec_unpacku_low ty x) (vec_rr ty (vecop_unpacku_low ty) x))
+
+(decl vecop_unpacku_high (Type) VecUnaryOp)
+(rule (vecop_unpacku_high $I8X16) (VecUnaryOp.UnpackUHigh8x16))
+(rule (vecop_unpacku_high $I16X8) (VecUnaryOp.UnpackUHigh16x8))
+(rule (vecop_unpacku_high $I32X4) (VecUnaryOp.UnpackUHigh32x4))
+
+(decl vec_unpacku_high (Type Reg) Reg)
+(rule (vec_unpacku_high ty x) (vec_rr ty (vecop_unpacku_high ty) x))
+
+
+;; Helpers for generating vector merge instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl vecop_merge_low (Type) VecBinaryOp)
+(rule (vecop_merge_low $I8X16) (VecBinaryOp.MergeLow8x16))
+(rule (vecop_merge_low $I16X8) (VecBinaryOp.MergeLow16x8))
+(rule (vecop_merge_low $I32X4) (VecBinaryOp.MergeLow32x4))
+(rule (vecop_merge_low $I64X2) (VecBinaryOp.MergeLow64x2))
+
+(decl vec_merge_low (Type Reg Reg) Reg)
+(rule (vec_merge_low ty x y) (vec_rrr ty (vecop_merge_low ty) x y))
+
+(decl vecop_merge_high (Type) VecBinaryOp)
+(rule (vecop_merge_high $I8X16) (VecBinaryOp.MergeHigh8x16))
+(rule (vecop_merge_high $I16X8) (VecBinaryOp.MergeHigh16x8))
+(rule (vecop_merge_high $I32X4) (VecBinaryOp.MergeHigh32x4))
+(rule (vecop_merge_high $I64X2) (VecBinaryOp.MergeHigh64x2))
+
+(decl vec_merge_high (Type Reg Reg) Reg)
+(rule (vec_merge_high ty x y) (vec_rrr ty (vecop_merge_high ty) x y))
+
+
;; Helpers for generating `clz` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Count leading zeroes. For a zero input, return the specified value.
@@ -2711,6 +3445,15 @@
(decl add_mem_sext32 (Type Reg MemArg) Reg)
(rule (add_mem_sext32 ty x y) (alu_rx ty (aluop_add_sext32 ty) x y))
+(decl vecop_add (Type) VecBinaryOp)
+(rule (vecop_add $I8X16) (VecBinaryOp.Add8x16))
+(rule (vecop_add $I16X8) (VecBinaryOp.Add16x8))
+(rule (vecop_add $I32X4) (VecBinaryOp.Add32x4))
+(rule (vecop_add $I64X2) (VecBinaryOp.Add64x2))
+
+(decl vec_add (Type Reg Reg) Reg)
+(rule (vec_add ty x y) (vec_rrr ty (vecop_add ty) x y))
+
;; Helpers for generating `add_logical` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2768,6 +3511,15 @@
(decl sub_mem_sext32 (Type Reg MemArg) Reg)
(rule (sub_mem_sext32 ty x y) (alu_rx ty (aluop_sub_sext32 ty) x y))
+(decl vecop_sub (Type) VecBinaryOp)
+(rule (vecop_sub $I8X16) (VecBinaryOp.Sub8x16))
+(rule (vecop_sub $I16X8) (VecBinaryOp.Sub16x8))
+(rule (vecop_sub $I32X4) (VecBinaryOp.Sub32x4))
+(rule (vecop_sub $I64X2) (VecBinaryOp.Sub64x2))
+
+(decl vec_sub (Type Reg Reg) Reg)
+(rule (vec_sub ty x y) (vec_rrr ty (vecop_sub ty) x y))
+
;; Helpers for generating `sub_logical` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2831,6 +3583,69 @@
(decl mul_mem_sext32 (Type Reg MemArg) Reg)
(rule (mul_mem_sext32 ty x y) (alu_rx ty (aluop_mul_sext32 ty) x y))
+(decl vecop_mul (Type) VecBinaryOp)
+(rule (vecop_mul $I8X16) (VecBinaryOp.Mul8x16))
+(rule (vecop_mul $I16X8) (VecBinaryOp.Mul16x8))
+(rule (vecop_mul $I32X4) (VecBinaryOp.Mul32x4))
+;; No support for $I64X2 multiplication.
+
+(decl vec_mul (Type Reg Reg) Reg)
+(rule (vec_mul ty x y) (vec_rrr ty (vecop_mul ty) x y))
+
+(decl vecop_umulhi (Type) VecBinaryOp)
+(rule (vecop_umulhi $I8X16) (VecBinaryOp.UMulHi8x16))
+(rule (vecop_umulhi $I16X8) (VecBinaryOp.UMulHi16x8))
+(rule (vecop_umulhi $I32X4) (VecBinaryOp.UMulHi32x4))
+;; No support for $I64X2 multiplication.
+
+(decl vec_umulhi (Type Reg Reg) Reg)
+(rule (vec_umulhi ty x y) (vec_rrr ty (vecop_umulhi ty) x y))
+
+(decl vecop_smulhi (Type) VecBinaryOp)
+(rule (vecop_smulhi $I8X16) (VecBinaryOp.SMulHi8x16))
+(rule (vecop_smulhi $I16X8) (VecBinaryOp.SMulHi16x8))
+(rule (vecop_smulhi $I32X4) (VecBinaryOp.SMulHi32x4))
+;; No support for $I64X2 multiplication.
+
+(decl vec_smulhi (Type Reg Reg) Reg)
+(rule (vec_smulhi ty x y) (vec_rrr ty (vecop_smulhi ty) x y))
+
+(decl vecop_umul_even (Type) VecBinaryOp)
+(rule (vecop_umul_even $I8X16) (VecBinaryOp.UMulEven8x16))
+(rule (vecop_umul_even $I16X8) (VecBinaryOp.UMulEven16x8))
+(rule (vecop_umul_even $I32X4) (VecBinaryOp.UMulEven32x4))
+;; No support for $I64X2 multiplication.
+
+(decl vec_umul_even (Type Reg Reg) Reg)
+(rule (vec_umul_even ty x y) (vec_rrr ty (vecop_umul_even ty) x y))
+
+(decl vecop_smul_even (Type) VecBinaryOp)
+(rule (vecop_smul_even $I8X16) (VecBinaryOp.SMulEven8x16))
+(rule (vecop_smul_even $I16X8) (VecBinaryOp.SMulEven16x8))
+(rule (vecop_smul_even $I32X4) (VecBinaryOp.SMulEven32x4))
+;; No support for $I64X2 multiplication.
+
+(decl vec_smul_even (Type Reg Reg) Reg)
+(rule (vec_smul_even ty x y) (vec_rrr ty (vecop_smul_even ty) x y))
+
+(decl vecop_umul_odd (Type) VecBinaryOp)
+(rule (vecop_umul_odd $I8X16) (VecBinaryOp.UMulOdd8x16))
+(rule (vecop_umul_odd $I16X8) (VecBinaryOp.UMulOdd16x8))
+(rule (vecop_umul_odd $I32X4) (VecBinaryOp.UMulOdd32x4))
+;; No support for $I64X2 multiplication.
+
+(decl vec_umul_odd (Type Reg Reg) Reg)
+(rule (vec_umul_odd ty x y) (vec_rrr ty (vecop_umul_odd ty) x y))
+
+(decl vecop_smul_odd (Type) VecBinaryOp)
+(rule (vecop_smul_odd $I8X16) (VecBinaryOp.SMulOdd8x16))
+(rule (vecop_smul_odd $I16X8) (VecBinaryOp.SMulOdd16x8))
+(rule (vecop_smul_odd $I32X4) (VecBinaryOp.SMulOdd32x4))
+;; No support for $I64X2 multiplication.
+
+(decl vec_smul_odd (Type Reg Reg) Reg)
+(rule (vec_smul_odd ty x y) (vec_rrr ty (vecop_smul_odd ty) x y))
+
;; Helpers for generating `udivmod` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2846,6 +3661,66 @@
(rule (sdivmod $I64 x y) (sdivmod64 x y))
+;; Helpers for generating `umax` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl vecop_umax (Type) VecBinaryOp)
+(rule (vecop_umax $I8X16) (VecBinaryOp.UMax8x16))
+(rule (vecop_umax $I16X8) (VecBinaryOp.UMax16x8))
+(rule (vecop_umax $I32X4) (VecBinaryOp.UMax32x4))
+(rule (vecop_umax $I64X2) (VecBinaryOp.UMax64x2))
+
+(decl vec_umax (Type Reg Reg) Reg)
+(rule (vec_umax ty x y) (vec_rrr ty (vecop_umax ty) x y))
+
+
+;; Helpers for generating `imax` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl vecop_smax (Type) VecBinaryOp)
+(rule (vecop_smax $I8X16) (VecBinaryOp.SMax8x16))
+(rule (vecop_smax $I16X8) (VecBinaryOp.SMax16x8))
+(rule (vecop_smax $I32X4) (VecBinaryOp.SMax32x4))
+(rule (vecop_smax $I64X2) (VecBinaryOp.SMax64x2))
+
+(decl vec_smax (Type Reg Reg) Reg)
+(rule (vec_smax ty x y) (vec_rrr ty (vecop_smax ty) x y))
+
+
+;; Helpers for generating `umin` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl vecop_umin (Type) VecBinaryOp)
+(rule (vecop_umin $I8X16) (VecBinaryOp.UMin8x16))
+(rule (vecop_umin $I16X8) (VecBinaryOp.UMin16x8))
+(rule (vecop_umin $I32X4) (VecBinaryOp.UMin32x4))
+(rule (vecop_umin $I64X2) (VecBinaryOp.UMin64x2))
+
+(decl vec_umin (Type Reg Reg) Reg)
+(rule (vec_umin ty x y) (vec_rrr ty (vecop_umin ty) x y))
+
+
+;; Helpers for generating `imin` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl vecop_smin (Type) VecBinaryOp)
+(rule (vecop_smin $I8X16) (VecBinaryOp.SMin8x16))
+(rule (vecop_smin $I16X8) (VecBinaryOp.SMin16x8))
+(rule (vecop_smin $I32X4) (VecBinaryOp.SMin32x4))
+(rule (vecop_smin $I64X2) (VecBinaryOp.SMin64x2))
+
+(decl vec_smin (Type Reg Reg) Reg)
+(rule (vec_smin ty x y) (vec_rrr ty (vecop_smin ty) x y))
+
+
+;; Helpers for generating `avg_round` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl vecop_uavg (Type) VecBinaryOp)
+(rule (vecop_uavg $I8X16) (VecBinaryOp.UAvg8x16))
+(rule (vecop_uavg $I16X8) (VecBinaryOp.UAvg16x8))
+(rule (vecop_uavg $I32X4) (VecBinaryOp.UAvg32x4))
+(rule (vecop_uavg $I64X2) (VecBinaryOp.UAvg64x2))
+
+(decl vec_uavg (Type Reg Reg) Reg)
+(rule (vec_uavg ty x y) (vec_rrr ty (vecop_uavg ty) x y))
+
+
;; Helpers for generating `and` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl aluop_and (Type) ALUOp)
@@ -2864,6 +3739,9 @@
(decl and_mem (Type Reg MemArg) Reg)
(rule (and_mem ty x y) (alu_rx ty (aluop_and ty) x y))
+(decl vec_and (Type Reg Reg) Reg)
+(rule (vec_and (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.And128) x y))
+
;; Helpers for generating `or` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2883,6 +3761,9 @@
(decl or_mem (Type Reg MemArg) Reg)
(rule (or_mem ty x y) (alu_rx ty (aluop_or ty) x y))
+(decl vec_or (Type Reg Reg) Reg)
+(rule (vec_or (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.Orr128) x y))
+
;; Helpers for generating `xor` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2903,6 +3784,10 @@
(rule (push_xor_uimm32shifted ib ty dst src imm)
(push_alu_uimm32shifted ib (aluop_xor ty) dst src imm))
+(decl vec_xor (Type Reg Reg) Reg)
+(rule (vec_xor (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.Xor128) x y))
+
+
;; Helpers for generating `not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl not_reg (Type Reg) Reg)
@@ -2920,6 +3805,9 @@
(let ((val Reg (push_xor_uimm32shifted ib ty dst src (uimm32shifted 0xffffffff 0))))
(push_xor_uimm32shifted ib ty dst val (uimm32shifted 0xffffffff 32))))
+(decl vec_not (Type Reg) Reg)
+(rule (vec_not ty x) (vec_not_or ty x x))
+
;; Helpers for generating `not_and` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2930,6 +3818,9 @@
(decl not_and_reg (Type Reg Reg) Reg)
(rule (not_and_reg ty x y) (alu_rrr ty (aluop_not_and ty) x y))
+(decl vec_not_and (Type Reg Reg) Reg)
+(rule (vec_not_and (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.NotAnd128) x y))
+
;; Helpers for generating `not_or` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2940,6 +3831,9 @@
(decl not_or_reg (Type Reg Reg) Reg)
(rule (not_or_reg ty x y) (alu_rrr ty (aluop_not_or ty) x y))
+(decl vec_not_or (Type Reg Reg) Reg)
+(rule (vec_not_or (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.NotOrr128) x y))
+
;; Helpers for generating `not_xor` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2950,6 +3844,9 @@
(decl not_xor_reg (Type Reg Reg) Reg)
(rule (not_xor_reg ty x y) (alu_rrr ty (aluop_not_xor ty) x y))
+(decl vec_not_xor (Type Reg Reg) Reg)
+(rule (vec_not_xor (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.NotXor128) x y))
+
;; Helpers for generating `and_not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2960,6 +3857,9 @@
(decl and_not_reg (Type Reg Reg) Reg)
(rule (and_not_reg ty x y) (alu_rrr ty (aluop_and_not ty) x y))
+(decl vec_and_not (Type Reg Reg) Reg)
+(rule (vec_and_not (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.AndNot128) x y))
+
;; Helpers for generating `or_not` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2970,6 +3870,15 @@
(decl or_not_reg (Type Reg Reg) Reg)
(rule (or_not_reg ty x y) (alu_rrr ty (aluop_or_not ty) x y))
+(decl vec_or_not (Type Reg Reg) Reg)
+(rule (vec_or_not (ty_vec128 ty) x y) (vec_rrr ty (VecBinaryOp.OrrNot128) x y))
+
+
+;; Helpers for generating `bitpermute` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl vec_bitpermute (Reg Reg) Reg)
+(rule (vec_bitpermute x y) (vec_rrr $I64X2 (VecBinaryOp.BitPermute128) x y))
+
;; Helpers for generating `abs` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2986,6 +3895,15 @@
(decl abs_reg_sext32 (Type Reg) Reg)
(rule (abs_reg_sext32 ty x) (unary_rr ty (unaryop_abs_sext32 ty) x))
+(decl vecop_abs (Type) VecUnaryOp)
+(rule (vecop_abs $I8X16) (VecUnaryOp.Abs8x16))
+(rule (vecop_abs $I16X8) (VecUnaryOp.Abs16x8))
+(rule (vecop_abs $I32X4) (VecUnaryOp.Abs32x4))
+(rule (vecop_abs $I64X2) (VecUnaryOp.Abs64x2))
+
+(decl vec_abs (Type Reg) Reg)
+(rule (vec_abs ty x) (vec_rr ty (vecop_abs ty) x))
+
;; Helpers for generating `neg` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3004,6 +3922,15 @@
(decl neg_reg_sext32 (Type Reg) Reg)
(rule (neg_reg_sext32 ty x) (unary_rr ty (unaryop_neg_sext32 ty) x))
+(decl vecop_neg (Type) VecUnaryOp)
+(rule (vecop_neg $I8X16) (VecUnaryOp.Neg8x16))
+(rule (vecop_neg $I16X8) (VecUnaryOp.Neg16x8))
+(rule (vecop_neg $I32X4) (VecUnaryOp.Neg32x4))
+(rule (vecop_neg $I64X2) (VecUnaryOp.Neg64x2))
+
+(decl vec_neg (Type Reg) Reg)
+(rule (vec_neg ty x) (vec_rr ty (vecop_neg ty) x))
+
;; Helpers for generating `bswap` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3040,6 +3967,20 @@
(rule (push_rot_imm_reg ib ty dst src shift_imm shift_reg)
(push_shift ib (shiftop_rot ty) dst src shift_imm shift_reg))
+(decl vec_shiftop_rot (Type) VecShiftOp)
+(rule (vec_shiftop_rot $I8X16) (VecShiftOp.RotL8x16))
+(rule (vec_shiftop_rot $I16X8) (VecShiftOp.RotL16x8))
+(rule (vec_shiftop_rot $I32X4) (VecShiftOp.RotL32x4))
+(rule (vec_shiftop_rot $I64X2) (VecShiftOp.RotL64x2))
+
+(decl vec_rot_reg (Type Reg Reg) Reg)
+(rule (vec_rot_reg ty x shift_reg)
+ (vec_shift_rr ty (vec_shiftop_rot ty) x 0 shift_reg))
+
+(decl vec_rot_imm (Type Reg u8) Reg)
+(rule (vec_rot_imm ty x shift_imm)
+ (vec_shift_rr ty (vec_shiftop_rot ty) x shift_imm (zero_reg)))
+
;; Helpers for generating `lshl` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3057,6 +3998,23 @@
(rule (lshl_imm ty x shift_imm)
(shift_rr ty (shiftop_lshl ty) x shift_imm (zero_reg)))
+(decl vec_shiftop_lshl (Type) VecShiftOp)
+(rule (vec_shiftop_lshl $I8X16) (VecShiftOp.LShL8x16))
+(rule (vec_shiftop_lshl $I16X8) (VecShiftOp.LShL16x8))
+(rule (vec_shiftop_lshl $I32X4) (VecShiftOp.LShL32x4))
+(rule (vec_shiftop_lshl $I64X2) (VecShiftOp.LShL64x2))
+
+(decl vec_lshl_reg (Type Reg Reg) Reg)
+(rule (vec_lshl_reg ty x shift_reg)
+ (vec_shift_rr ty (vec_shiftop_lshl ty) x 0 shift_reg))
+
+(decl vec_lshl_imm (Type Reg u8) Reg)
+(rule (vec_lshl_imm ty x shift_imm)
+ (vec_shift_rr ty (vec_shiftop_lshl ty) x shift_imm (zero_reg)))
+
+(decl vec_lshl_by_byte (Reg Reg) Reg)
+(rule (vec_lshl_by_byte x y) (vec_rrr $I8X16 (VecBinaryOp.LShLByByte128) x y))
+
;; Helpers for generating `lshr` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3072,6 +4030,23 @@
(rule (lshr_imm ty x shift_imm)
(shift_rr ty (shiftop_lshr ty) x shift_imm (zero_reg)))
+(decl vec_shiftop_lshr (Type) VecShiftOp)
+(rule (vec_shiftop_lshr $I8X16) (VecShiftOp.LShR8x16))
+(rule (vec_shiftop_lshr $I16X8) (VecShiftOp.LShR16x8))
+(rule (vec_shiftop_lshr $I32X4) (VecShiftOp.LShR32x4))
+(rule (vec_shiftop_lshr $I64X2) (VecShiftOp.LShR64x2))
+
+(decl vec_lshr_reg (Type Reg Reg) Reg)
+(rule (vec_lshr_reg ty x shift_reg)
+ (vec_shift_rr ty (vec_shiftop_lshr ty) x 0 shift_reg))
+
+(decl vec_lshr_imm (Type Reg u8) Reg)
+(rule (vec_lshr_imm ty x shift_imm)
+ (vec_shift_rr ty (vec_shiftop_lshr ty) x shift_imm (zero_reg)))
+
+(decl vec_lshr_by_byte (Reg Reg) Reg)
+(rule (vec_lshr_by_byte x y) (vec_rrr $I8X16 (VecBinaryOp.LShRByByte128) x y))
+
;; Helpers for generating `ashr` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3087,6 +4062,23 @@
(rule (ashr_imm ty x shift_imm)
(shift_rr ty (shiftop_ashr ty) x shift_imm (zero_reg)))
+(decl vec_shiftop_ashr (Type) VecShiftOp)
+(rule (vec_shiftop_ashr $I8X16) (VecShiftOp.AShR8x16))
+(rule (vec_shiftop_ashr $I16X8) (VecShiftOp.AShR16x8))
+(rule (vec_shiftop_ashr $I32X4) (VecShiftOp.AShR32x4))
+(rule (vec_shiftop_ashr $I64X2) (VecShiftOp.AShR64x2))
+
+(decl vec_ashr_reg (Type Reg Reg) Reg)
+(rule (vec_ashr_reg ty x shift_reg)
+ (vec_shift_rr ty (vec_shiftop_ashr ty) x 0 shift_reg))
+
+(decl vec_ashr_imm (Type Reg u8) Reg)
+(rule (vec_ashr_imm ty x shift_imm)
+ (vec_shift_rr ty (vec_shiftop_ashr ty) x shift_imm (zero_reg)))
+
+(decl vec_ashr_by_byte (Reg Reg) Reg)
+(rule (vec_ashr_by_byte x y) (vec_rrr $I8X16 (VecBinaryOp.AShRByByte128) x y))
+
;; Helpers for generating `popcnt` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3096,6 +4088,15 @@
(decl popcnt_reg (Reg) Reg)
(rule (popcnt_reg x) (unary_rr $I64 (UnaryOp.PopcntReg) x))
+(decl vecop_popcnt (Type) VecUnaryOp)
+(rule (vecop_popcnt $I8X16) (VecUnaryOp.Popcnt8x16))
+(rule (vecop_popcnt $I16X8) (VecUnaryOp.Popcnt16x8))
+(rule (vecop_popcnt $I32X4) (VecUnaryOp.Popcnt32x4))
+(rule (vecop_popcnt $I64X2) (VecUnaryOp.Popcnt64x2))
+
+(decl vec_popcnt (Type Reg) Reg)
+(rule (vec_popcnt ty x) (vec_rr ty (vecop_popcnt ty) x))
+
;; Helpers for generating `atomic_rmw` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3132,6 +4133,8 @@
(decl fpuop2_add (Type) FPUOp2)
(rule (fpuop2_add $F32) (FPUOp2.Add32))
(rule (fpuop2_add $F64) (FPUOp2.Add64))
+(rule (fpuop2_add $F32X4) (FPUOp2.Add32x4))
+(rule (fpuop2_add $F64X2) (FPUOp2.Add64x2))
(decl fadd_reg (Type Reg Reg) Reg)
(rule (fadd_reg ty x y) (fpu_rrr ty (fpuop2_add ty) x y))
@@ -3142,6 +4145,8 @@
(decl fpuop2_sub (Type) FPUOp2)
(rule (fpuop2_sub $F32) (FPUOp2.Sub32))
(rule (fpuop2_sub $F64) (FPUOp2.Sub64))
+(rule (fpuop2_sub $F32X4) (FPUOp2.Sub32x4))
+(rule (fpuop2_sub $F64X2) (FPUOp2.Sub64x2))
(decl fsub_reg (Type Reg Reg) Reg)
(rule (fsub_reg ty x y) (fpu_rrr ty (fpuop2_sub ty) x y))
@@ -3152,6 +4157,8 @@
(decl fpuop2_mul (Type) FPUOp2)
(rule (fpuop2_mul $F32) (FPUOp2.Mul32))
(rule (fpuop2_mul $F64) (FPUOp2.Mul64))
+(rule (fpuop2_mul $F32X4) (FPUOp2.Mul32x4))
+(rule (fpuop2_mul $F64X2) (FPUOp2.Mul64x2))
(decl fmul_reg (Type Reg Reg) Reg)
(rule (fmul_reg ty x y) (fpu_rrr ty (fpuop2_mul ty) x y))
@@ -3162,6 +4169,8 @@
(decl fpuop2_div (Type) FPUOp2)
(rule (fpuop2_div $F32) (FPUOp2.Div32))
(rule (fpuop2_div $F64) (FPUOp2.Div64))
+(rule (fpuop2_div $F32X4) (FPUOp2.Div32x4))
+(rule (fpuop2_div $F64X2) (FPUOp2.Div64x2))
(decl fdiv_reg (Type Reg Reg) Reg)
(rule (fdiv_reg ty x y) (fpu_rrr ty (fpuop2_div ty) x y))
@@ -3172,6 +4181,8 @@
(decl fpuop2_min (Type) FPUOp2)
(rule (fpuop2_min $F32) (FPUOp2.Min32))
(rule (fpuop2_min $F64) (FPUOp2.Min64))
+(rule (fpuop2_min $F32X4) (FPUOp2.Min32x4))
+(rule (fpuop2_min $F64X2) (FPUOp2.Min64x2))
(decl fmin_reg (Type Reg Reg) Reg)
(rule (fmin_reg ty x y) (fpu_rrr ty (fpuop2_min ty) x y))
@@ -3182,16 +4193,44 @@
(decl fpuop2_max (Type) FPUOp2)
(rule (fpuop2_max $F32) (FPUOp2.Max32))
(rule (fpuop2_max $F64) (FPUOp2.Max64))
+(rule (fpuop2_max $F32X4) (FPUOp2.Max32x4))
+(rule (fpuop2_max $F64X2) (FPUOp2.Max64x2))
(decl fmax_reg (Type Reg Reg) Reg)
(rule (fmax_reg ty x y) (fpu_rrr ty (fpuop2_max ty) x y))
+;; Helpers for generating `fmin_pseudo` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop2_min_pseudo (Type) FPUOp2)
+(rule (fpuop2_min_pseudo $F32) (FPUOp2.MinPseudo32))
+(rule (fpuop2_min_pseudo $F64) (FPUOp2.MinPseudo64))
+(rule (fpuop2_min_pseudo $F32X4) (FPUOp2.MinPseudo32x4))
+(rule (fpuop2_min_pseudo $F64X2) (FPUOp2.MinPseudo64x2))
+
+(decl fmin_pseudo_reg (Type Reg Reg) Reg)
+(rule (fmin_pseudo_reg ty x y) (fpu_rrr ty (fpuop2_min_pseudo ty) x y))
+
+
+;; Helpers for generating `fmax_pseudo` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl fpuop2_max_pseudo (Type) FPUOp2)
+(rule (fpuop2_max_pseudo $F32) (FPUOp2.MaxPseudo32))
+(rule (fpuop2_max_pseudo $F64) (FPUOp2.MaxPseudo64))
+(rule (fpuop2_max_pseudo $F32X4) (FPUOp2.MaxPseudo32x4))
+(rule (fpuop2_max_pseudo $F64X2) (FPUOp2.MaxPseudo64x2))
+
+(decl fmax_pseudo_reg (Type Reg Reg) Reg)
+(rule (fmax_pseudo_reg ty x y) (fpu_rrr ty (fpuop2_max_pseudo ty) x y))
+
+
;; Helpers for generating `fma` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl fpuop3_fma (Type) FPUOp3)
(rule (fpuop3_fma $F32) (FPUOp3.MAdd32))
(rule (fpuop3_fma $F64) (FPUOp3.MAdd64))
+(rule (fpuop3_fma $F32X4) (FPUOp3.MAdd32x4))
+(rule (fpuop3_fma $F64X2) (FPUOp3.MAdd64x2))
(decl fma_reg (Type Reg Reg Reg) Reg)
(rule (fma_reg ty x y acc) (fpu_rrrr ty (fpuop3_fma ty) x y acc))
@@ -3202,6 +4241,8 @@
(decl fpuop1_sqrt (Type) FPUOp1)
(rule (fpuop1_sqrt $F32) (FPUOp1.Sqrt32))
(rule (fpuop1_sqrt $F64) (FPUOp1.Sqrt64))
+(rule (fpuop1_sqrt $F32X4) (FPUOp1.Sqrt32x4))
+(rule (fpuop1_sqrt $F64X2) (FPUOp1.Sqrt64x2))
(decl sqrt_reg (Type Reg) Reg)
(rule (sqrt_reg ty x) (fpu_rr ty (fpuop1_sqrt ty) x))
@@ -3212,6 +4253,8 @@
(decl fpuop1_neg (Type) FPUOp1)
(rule (fpuop1_neg $F32) (FPUOp1.Neg32))
(rule (fpuop1_neg $F64) (FPUOp1.Neg64))
+(rule (fpuop1_neg $F32X4) (FPUOp1.Neg32x4))
+(rule (fpuop1_neg $F64X2) (FPUOp1.Neg64x2))
(decl fneg_reg (Type Reg) Reg)
(rule (fneg_reg ty x) (fpu_rr ty (fpuop1_neg ty) x))
@@ -3222,6 +4265,8 @@
(decl fpuop1_abs (Type) FPUOp1)
(rule (fpuop1_abs $F32) (FPUOp1.Abs32))
(rule (fpuop1_abs $F64) (FPUOp1.Abs64))
+(rule (fpuop1_abs $F32X4) (FPUOp1.Abs32x4))
+(rule (fpuop1_abs $F64X2) (FPUOp1.Abs64x2))
(decl fabs_reg (Type Reg) Reg)
(rule (fabs_reg ty x) (fpu_rr ty (fpuop1_abs ty) x))
@@ -3232,6 +4277,8 @@
(decl fpuroundop_round (Type) FpuRoundOp)
(rule (fpuroundop_round $F32) (FpuRoundOp.Round32))
(rule (fpuroundop_round $F64) (FpuRoundOp.Round64))
+(rule (fpuroundop_round $F32X4) (FpuRoundOp.Round32x4))
+(rule (fpuroundop_round $F64X2) (FpuRoundOp.Round64x2))
(decl ceil_reg (Type Reg) Reg)
(rule (ceil_reg ty x) (fpu_round ty (fpuroundop_round ty)
@@ -3256,6 +4303,8 @@
(rule (fpromote_reg ty ty x) x)
(rule (fpromote_reg $F64 $F32 x)
(fpu_rr $F64 (FPUOp1.Cvt32To64) x))
+(rule (fpromote_reg $F64X2 $F32X4 x)
+ (fpu_rr $F64 (FPUOp1.Cvt32x4To64x2) x))
;; Helpers for generating `fdemote` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3264,28 +4313,34 @@
(rule (fdemote_reg ty ty mode x) x)
(rule (fdemote_reg $F32 $F64 mode x)
(fpu_round $F32 (FpuRoundOp.Cvt64To32) mode x))
+(rule (fdemote_reg $F32X4 $F64X2 mode x)
+ (fpu_round $F32X4 (FpuRoundOp.Cvt64x2To32x4) mode x))
;; Helpers for generating `fcvt_from_uint` instructions ;;;;;;;;;;;;;;;;;;;;;;;;
-(decl uint_to_fpu_op (Type) FpuRoundOp)
-(rule (uint_to_fpu_op $F32) (FpuRoundOp.FromUInt32))
-(rule (uint_to_fpu_op $F64) (FpuRoundOp.FromUInt64))
-
(decl fcvt_from_uint_reg (Type FpuRoundMode Reg) Reg)
-(rule (fcvt_from_uint_reg ty mode x)
- (fpu_round ty (uint_to_fpu_op ty) mode x))
+(rule (fcvt_from_uint_reg $F32 mode x)
+ (fpu_round $F32 (FpuRoundOp.FromUInt32) mode (vec_insert_lane_undef $I32X4 x 0 (zero_reg))))
+(rule (fcvt_from_uint_reg $F64 mode x)
+ (fpu_round $F64 (FpuRoundOp.FromUInt64) mode (vec_insert_lane_undef $I64X2 x 0 (zero_reg))))
+(rule (fcvt_from_uint_reg $F32X4 mode x)
+ (fpu_round $F32X4 (FpuRoundOp.FromUInt32x4) mode x))
+(rule (fcvt_from_uint_reg $F64X2 mode x)
+ (fpu_round $F64X2 (FpuRoundOp.FromUInt64x2) mode x))
;; Helpers for generating `fcvt_from_sint` instructions ;;;;;;;;;;;;;;;;;;;;;;;;
-(decl sint_to_fpu_op (Type) FpuRoundOp)
-(rule (sint_to_fpu_op $F32) (FpuRoundOp.FromSInt32))
-(rule (sint_to_fpu_op $F64) (FpuRoundOp.FromSInt64))
-
(decl fcvt_from_sint_reg (Type FpuRoundMode Reg) Reg)
-(rule (fcvt_from_sint_reg ty mode x)
- (fpu_round ty (sint_to_fpu_op ty) mode x))
+(rule (fcvt_from_sint_reg $F32 mode x)
+ (fpu_round $F32 (FpuRoundOp.FromSInt32) mode (vec_insert_lane_undef $I32X4 x 0 (zero_reg))))
+(rule (fcvt_from_sint_reg $F64 mode x)
+ (fpu_round $F64 (FpuRoundOp.FromSInt64) mode (vec_insert_lane_undef $I64X2 x 0 (zero_reg))))
+(rule (fcvt_from_sint_reg $F32X4 mode x)
+ (fpu_round $F32X4 (FpuRoundOp.FromSInt32x4) mode x))
+(rule (fcvt_from_sint_reg $F64X2 mode x)
+ (fpu_round $F64X2 (FpuRoundOp.FromSInt64x2) mode x))
;; Helpers for generating `fcvt_to_[us]int` instructions ;;;;;;;;;;;;;;;;;;;;;;;
@@ -3305,9 +4360,13 @@
(decl fcvt_to_uint_reg (Type FpuRoundMode Reg) Reg)
(rule (fcvt_to_uint_reg $F32 mode x)
- (mov_from_fpr32 (fpu_round $F32 (FpuRoundOp.ToUInt32) mode x)))
+ (vec_extract_lane $I32X4 (fpu_round $F32 (FpuRoundOp.ToUInt32) mode x) 0 (zero_reg)))
(rule (fcvt_to_uint_reg $F64 mode x)
- (mov_from_fpr64 (fpu_round $F64 (FpuRoundOp.ToUInt64) mode x)))
+ (vec_extract_lane $I64X2 (fpu_round $F64 (FpuRoundOp.ToUInt64) mode x) 0 (zero_reg)))
+(rule (fcvt_to_uint_reg $F32X4 mode x)
+ (fpu_round $F32X4 (FpuRoundOp.ToUInt32x4) mode x))
+(rule (fcvt_to_uint_reg $F64X2 mode x)
+ (fpu_round $F64X2 (FpuRoundOp.ToUInt64x2) mode x))
(decl fcvt_to_uint_ub (Type Type) Reg)
(rule (fcvt_to_uint_ub $F32 dst_ty)
@@ -3333,9 +4392,13 @@
(decl fcvt_to_sint_reg (Type FpuRoundMode Reg) Reg)
(rule (fcvt_to_sint_reg $F32 mode x)
- (mov_from_fpr32 (fpu_round $F32 (FpuRoundOp.ToSInt32) mode x)))
+ (vec_extract_lane $F32X4 (fpu_round $F32 (FpuRoundOp.ToSInt32) mode x) 0 (zero_reg)))
(rule (fcvt_to_sint_reg $F64 mode x)
- (mov_from_fpr64 (fpu_round $F64 (FpuRoundOp.ToSInt64) mode x)))
+ (vec_extract_lane $F64X2 (fpu_round $F64 (FpuRoundOp.ToSInt64) mode x) 0 (zero_reg)))
+(rule (fcvt_to_sint_reg $F32X4 mode x)
+ (fpu_round $F32X4 (FpuRoundOp.ToSInt32x4) mode x))
+(rule (fcvt_to_sint_reg $F64X2 mode x)
+ (fpu_round $F64X2 (FpuRoundOp.ToSInt64x2) mode x))
(decl fcvt_to_sint_ub (Type Type) Reg)
(rule (fcvt_to_sint_ub $F32 dst_ty)
@@ -3426,12 +4489,79 @@
(rule (icmpu_mem_zext32 ty src mem) (cmp_rx (cmpop_cmpu_zext32 ty) src mem))
+;; Helpers for generating vector `icmp` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl vecop_int_cmpeq (Type) VecIntCmpOp)
+(rule (vecop_int_cmpeq (multi_lane 8 16)) (VecIntCmpOp.CmpEq8x16))
+(rule (vecop_int_cmpeq (multi_lane 16 8)) (VecIntCmpOp.CmpEq16x8))
+(rule (vecop_int_cmpeq (multi_lane 32 4)) (VecIntCmpOp.CmpEq32x4))
+(rule (vecop_int_cmpeq (multi_lane 64 2)) (VecIntCmpOp.CmpEq64x2))
+
+(decl vec_cmpeq (Type Reg Reg) Reg)
+(rule (vec_cmpeq (ty_vec128 ty) x y) (vec_int_cmp ty (vecop_int_cmpeq ty) x y))
+(decl vec_cmpeqs (Type Reg Reg) ProducesFlags)
+(rule (vec_cmpeqs (ty_vec128 ty) x y) (vec_int_cmps ty (vecop_int_cmpeq ty) x y))
+
+(decl vecop_int_cmph (Type) VecIntCmpOp)
+(rule (vecop_int_cmph (multi_lane 8 16)) (VecIntCmpOp.SCmpHi8x16))
+(rule (vecop_int_cmph (multi_lane 16 8)) (VecIntCmpOp.SCmpHi16x8))
+(rule (vecop_int_cmph (multi_lane 32 4)) (VecIntCmpOp.SCmpHi32x4))
+(rule (vecop_int_cmph (multi_lane 64 2)) (VecIntCmpOp.SCmpHi64x2))
+
+(decl vec_cmph (Type Reg Reg) Reg)
+(rule (vec_cmph (ty_vec128 ty) x y) (vec_int_cmp ty (vecop_int_cmph ty) x y))
+(decl vec_cmphs (Type Reg Reg) ProducesFlags)
+(rule (vec_cmphs (ty_vec128 ty) x y) (vec_int_cmps ty (vecop_int_cmph ty) x y))
+
+(decl vecop_int_cmphl (Type) VecIntCmpOp)
+(rule (vecop_int_cmphl (multi_lane 8 16)) (VecIntCmpOp.UCmpHi8x16))
+(rule (vecop_int_cmphl (multi_lane 16 8)) (VecIntCmpOp.UCmpHi16x8))
+(rule (vecop_int_cmphl (multi_lane 32 4)) (VecIntCmpOp.UCmpHi32x4))
+(rule (vecop_int_cmphl (multi_lane 64 2)) (VecIntCmpOp.UCmpHi64x2))
+
+(decl vec_cmphl (Type Reg Reg) Reg)
+(rule (vec_cmphl (ty_vec128 ty) x y) (vec_int_cmp ty (vecop_int_cmphl ty) x y))
+(decl vec_cmphls (Type Reg Reg) ProducesFlags)
+(rule (vec_cmphls (ty_vec128 ty) x y) (vec_int_cmps ty (vecop_int_cmphl ty) x y))
+
+
;; Helpers for generating `fcmp` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl fcmp_reg (Type Reg Reg) ProducesFlags)
(rule (fcmp_reg $F32 src1 src2) (fpu_cmp32 src1 src2))
(rule (fcmp_reg $F64 src1 src2) (fpu_cmp64 src1 src2))
+
+;; Helpers for generating vector `fcmp` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl vecop_float_cmpeq (Type) VecFloatCmpOp)
+(rule (vecop_float_cmpeq (multi_lane 32 4)) (VecFloatCmpOp.CmpEq32x4))
+(rule (vecop_float_cmpeq (multi_lane 64 2)) (VecFloatCmpOp.CmpEq64x2))
+
+(decl vec_fcmpeq (Type Reg Reg) Reg)
+(rule (vec_fcmpeq (ty_vec128 ty) x y) (vec_float_cmp ty (vecop_float_cmpeq ty) x y))
+(decl vec_fcmpeqs (Type Reg Reg) ProducesFlags)
+(rule (vec_fcmpeqs (ty_vec128 ty) x y) (vec_float_cmps ty (vecop_float_cmpeq ty) x y))
+
+(decl vecop_float_cmph (Type) VecFloatCmpOp)
+(rule (vecop_float_cmph (multi_lane 32 4)) (VecFloatCmpOp.CmpHi32x4))
+(rule (vecop_float_cmph (multi_lane 64 2)) (VecFloatCmpOp.CmpHi64x2))
+
+(decl vec_fcmph (Type Reg Reg) Reg)
+(rule (vec_fcmph (ty_vec128 ty) x y) (vec_float_cmp ty (vecop_float_cmph ty) x y))
+(decl vec_fcmphs (Type Reg Reg) ProducesFlags)
+(rule (vec_fcmphs (ty_vec128 ty) x y) (vec_float_cmps ty (vecop_float_cmph ty) x y))
+
+(decl vecop_float_cmphe (Type) VecFloatCmpOp)
+(rule (vecop_float_cmphe (multi_lane 32 4)) (VecFloatCmpOp.CmpHiEq32x4))
+(rule (vecop_float_cmphe (multi_lane 64 2)) (VecFloatCmpOp.CmpHiEq64x2))
+
+(decl vec_fcmphe (Type Reg Reg) Reg)
+(rule (vec_fcmphe (ty_vec128 ty) x y) (vec_float_cmp ty (vecop_float_cmphe ty) x y))
+(decl vec_fcmphes (Type Reg Reg) ProducesFlags)
+(rule (vec_fcmphes (ty_vec128 ty) x y) (vec_float_cmps ty (vecop_float_cmphe ty) x y))
+
+
;; Implicit conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(convert WritableRegPair RegPair writable_regpair_to_regpair)
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index a75e6ffaf334..3b7832b7ea58 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -852,6 +852,74 @@ fn enc_siy(opcode: u16, b1: Reg, d1: u32, i2: u8) -> [u8; 6] {
enc
}
+/// VRIa-type instructions.
+///
+/// 47 39 35 31 15 11 7
+/// opcode1 v1 - i2 m3 rxb opcode2
+/// 40 36 32 16 12 8 0
+///
+fn enc_vri_a(opcode: u16, v1: Reg, i2: u16, m3: u8) -> [u8; 6] {
+ let opcode1 = ((opcode >> 8) & 0xff) as u8;
+ let opcode2 = (opcode & 0xff) as u8;
+ let rxb = rxb(Some(v1), None, None, None);
+ let v1 = machreg_to_vr(v1) & 0x0f;
+ let m3 = m3 & 0x0f;
+
+ let mut enc: [u8; 6] = [0; 6];
+ enc[0] = opcode1;
+ enc[1] = v1 << 4;
+ enc[2..4].copy_from_slice(&i2.to_be_bytes());
+ enc[4] = m3 << 4 | rxb;
+ enc[5] = opcode2;
+ enc
+}
+
+/// VRIb-type instructions.
+///
+/// 47 39 35 31 23 15 11 7
+/// opcode1 v1 - i2 i3 m4 rxb opcode2
+/// 40 36 32 24 16 12 8 0
+///
+fn enc_vri_b(opcode: u16, v1: Reg, i2: u8, i3: u8, m4: u8) -> [u8; 6] {
+ let opcode1 = ((opcode >> 8) & 0xff) as u8;
+ let opcode2 = (opcode & 0xff) as u8;
+ let rxb = rxb(Some(v1), None, None, None);
+ let v1 = machreg_to_vr(v1) & 0x0f;
+ let m4 = m4 & 0x0f;
+
+ let mut enc: [u8; 6] = [0; 6];
+ enc[0] = opcode1;
+ enc[1] = v1 << 4;
+ enc[2] = i2;
+ enc[3] = i3;
+ enc[4] = m4 << 4 | rxb;
+ enc[5] = opcode2;
+ enc
+}
+
+/// VRIc-type instructions.
+///
+/// 47 39 35 31 15 11 7
+/// opcode1 v1 v3 i2 m4 rxb opcode2
+/// 40 36 32 16 12 8 0
+///
+fn enc_vri_c(opcode: u16, v1: Reg, i2: u16, v3: Reg, m4: u8) -> [u8; 6] {
+ let opcode1 = ((opcode >> 8) & 0xff) as u8;
+ let opcode2 = (opcode & 0xff) as u8;
+ let rxb = rxb(Some(v1), Some(v3), None, None);
+ let v1 = machreg_to_vr(v1) & 0x0f;
+ let v3 = machreg_to_vr(v3) & 0x0f;
+ let m4 = m4 & 0x0f;
+
+ let mut enc: [u8; 6] = [0; 6];
+ enc[0] = opcode1;
+ enc[1] = v1 << 4 | v3;
+ enc[2..4].copy_from_slice(&i2.to_be_bytes());
+ enc[4] = m4 << 4 | rxb;
+ enc[5] = opcode2;
+ enc
+}
+
/// VRRa-type instructions.
///
/// 47 39 35 31 23 19 15 11 7
@@ -878,6 +946,32 @@ fn enc_vrr_a(opcode: u16, v1: Reg, v2: Reg, m3: u8, m4: u8, m5: u8) -> [u8; 6] {
enc
}
+/// VRRb-type instructions.
+///
+/// 47 39 35 31 27 23 19 15 11 7
+/// opcode1 v1 v2 v3 - m5 - m4 rxb opcode2
+/// 40 36 32 28 24 20 16 12 8 0
+///
+fn enc_vrr_b(opcode: u16, v1: Reg, v2: Reg, v3: Reg, m4: u8, m5: u8) -> [u8; 6] {
+ let opcode1 = ((opcode >> 8) & 0xff) as u8;
+ let opcode2 = (opcode & 0xff) as u8;
+ let rxb = rxb(Some(v1), Some(v2), Some(v3), None);
+ let v1 = machreg_to_vr(v1) & 0x0f;
+ let v2 = machreg_to_vr(v2) & 0x0f;
+ let v3 = machreg_to_vr(v3) & 0x0f;
+ let m4 = m4 & 0x0f;
+ let m5 = m5 & 0x0f;
+
+ let mut enc: [u8; 6] = [0; 6];
+ enc[0] = opcode1;
+ enc[1] = v1 << 4 | v2;
+ enc[2] = v3 << 4;
+ enc[3] = m5 << 4;
+ enc[4] = m4 << 4 | rxb;
+ enc[5] = opcode2;
+ enc
+}
+
/// VRRc-type instructions.
///
/// 47 39 35 31 27 23 19 15 11 7
@@ -932,6 +1026,56 @@ fn enc_vrr_e(opcode: u16, v1: Reg, v2: Reg, v3: Reg, v4: Reg, m5: u8, m6: u8) ->
enc
}
+/// VRRf-type instructions.
+///
+/// 47 39 35 31 27 11 7
+/// opcode1 v1 r2 r3 - rxb opcode2
+/// 40 36 32 28 12 8 0
+///
+fn enc_vrr_f(opcode: u16, v1: Reg, r2: Reg, r3: Reg) -> [u8; 6] {
+ let opcode1 = ((opcode >> 8) & 0xff) as u8;
+ let opcode2 = (opcode & 0xff) as u8;
+ let rxb = rxb(Some(v1), None, None, None);
+ let v1 = machreg_to_vr(v1) & 0x0f;
+ let r2 = machreg_to_gpr(r2) & 0x0f;
+ let r3 = machreg_to_gpr(r3) & 0x0f;
+
+ let mut enc: [u8; 6] = [0; 6];
+ enc[0] = opcode1;
+ enc[1] = v1 << 4 | r2;
+ enc[2] = r3 << 4;
+ enc[4] = rxb;
+ enc[5] = opcode2;
+ enc
+}
+
+/// VRSa-type instructions.
+///
+/// 47 39 35 31 27 15 11 7
+/// opcode1 v1 v3 b2 d2 m4 rxb opcode2
+/// 40 36 32 28 16 12 8 0
+///
+fn enc_vrs_a(opcode: u16, v1: Reg, b2: Reg, d2: u32, v3: Reg, m4: u8) -> [u8; 6] {
+ let opcode1 = ((opcode >> 8) & 0xff) as u8;
+ let opcode2 = (opcode & 0xff) as u8;
+ let rxb = rxb(Some(v1), Some(v3), None, None);
+ let v1 = machreg_to_vr(v1) & 0x0f;
+ let b2 = machreg_to_gpr(b2) & 0x0f;
+ let v3 = machreg_to_vr(v3) & 0x0f;
+ let d2_lo = (d2 & 0xff) as u8;
+ let d2_hi = ((d2 >> 8) & 0x0f) as u8;
+ let m4 = m4 & 0x0f;
+
+ let mut enc: [u8; 6] = [0; 6];
+ enc[0] = opcode1;
+ enc[1] = v1 << 4 | v3;
+ enc[2] = b2 << 4 | d2_hi;
+ enc[3] = d2_lo;
+ enc[4] = m4 << 4 | rxb;
+ enc[5] = opcode2;
+ enc
+}
+
/// VRSb-type instructions.
///
/// 47 39 35 31 27 15 11 7
@@ -1834,29 +1978,6 @@ impl MachInstEmit for Inst {
rd, &mem, opcode_rx, opcode_rxy, opcode_ril, true, sink, emit_info, state,
);
}
- &Inst::FpuLoad32 { rd, ref mem }
- | &Inst::FpuLoad64 { rd, ref mem }
- | &Inst::FpuLoadRev32 { rd, ref mem }
- | &Inst::FpuLoadRev64 { rd, ref mem } => {
- let rd = allocs.next_writable(rd);
- let mem = mem.with_allocs(&mut allocs);
-
- let (opcode_rx, opcode_rxy, opcode_vrx) = match self {
- &Inst::FpuLoad32 { .. } => (Some(0x78), Some(0xed64), 0xe703), // LE(Y), VLEF
- &Inst::FpuLoad64 { .. } => (Some(0x68), Some(0xed65), 0xe702), // LD(Y), VLEG
- &Inst::FpuLoadRev32 { .. } => (None, None, 0xe603), // VLEBRF
- &Inst::FpuLoadRev64 { .. } => (None, None, 0xe602), // VLEBRG
- _ => unreachable!(),
- };
- let rd = rd.to_reg();
- if is_fpr(rd) && opcode_rx.is_some() {
- mem_emit(
- rd, &mem, opcode_rx, opcode_rxy, None, true, sink, emit_info, state,
- );
- } else {
- mem_vrx_emit(rd, &mem, opcode_vrx, 0, true, sink, emit_info, state);
- }
- }
&Inst::Store8 { rd, ref mem }
| &Inst::Store16 { rd, ref mem }
@@ -1904,28 +2025,6 @@ impl MachInstEmit for Inst {
};
mem_imm16_emit(imm, &mem, opcode, true, sink, emit_info, state);
}
- &Inst::FpuStore32 { rd, ref mem }
- | &Inst::FpuStore64 { rd, ref mem }
- | &Inst::FpuStoreRev32 { rd, ref mem }
- | &Inst::FpuStoreRev64 { rd, ref mem } => {
- let rd = allocs.next(rd);
- let mem = mem.with_allocs(&mut allocs);
-
- let (opcode_rx, opcode_rxy, opcode_vrx) = match self {
- &Inst::FpuStore32 { .. } => (Some(0x70), Some(0xed66), 0xe70b), // STE(Y), VSTEF
- &Inst::FpuStore64 { .. } => (Some(0x60), Some(0xed67), 0xe70a), // STD(Y), VSTEG
- &Inst::FpuStoreRev32 { .. } => (None, None, 0xe60b), // VSTEBRF
- &Inst::FpuStoreRev64 { .. } => (None, None, 0xe60a), // VSTEBRG
- _ => unreachable!(),
- };
- if is_fpr(rd) && opcode_rx.is_some() {
- mem_emit(
- rd, &mem, opcode_rx, opcode_rxy, None, true, sink, emit_info, state,
- );
- } else {
- mem_vrx_emit(rd, &mem, opcode_vrx, 0, true, sink, emit_info, state);
- }
- }
&Inst::LoadMultiple64 { rt, rt2, ref mem } => {
let mem = mem.with_allocs(&mut allocs);
@@ -2168,44 +2267,6 @@ impl MachInstEmit for Inst {
put(sink, &enc_vrr_a(opcode, rd.to_reg(), rm, 0, 0, 0));
}
}
- &Inst::MovToFpr32 { rd, rn } => {
- let rd = allocs.next_writable(rd);
- let rn = allocs.next(rn);
-
- let (opcode, m4) = (0xe722, 2); // VLVG
- put(sink, &enc_vrs_b(opcode, rd.to_reg(), zero_reg(), 0, rn, m4));
- }
- &Inst::MovToFpr64 { rd, rn } => {
- let rd = allocs.next_writable(rd);
- let rn = allocs.next(rn);
-
- if is_fpr(rd.to_reg()) {
- let opcode = 0xb3c1; // LDGR
- put(sink, &enc_rre(opcode, rd.to_reg(), rn));
- } else {
- let (opcode, m4) = (0xe722, 3); // VLVG
- put(sink, &enc_vrs_b(opcode, rd.to_reg(), zero_reg(), 0, rn, m4));
- }
- }
- &Inst::MovFromFpr32 { rd, rn } => {
- let rd = allocs.next_writable(rd);
- let rn = allocs.next(rn);
-
- let (opcode, m4) = (0xe721, 2); // VLGV
- put(sink, &enc_vrs_c(opcode, rd.to_reg(), zero_reg(), 0, rn, m4));
- }
- &Inst::MovFromFpr64 { rd, rn } => {
- let rd = allocs.next_writable(rd);
- let rn = allocs.next(rn);
-
- if is_fpr(rn) {
- let opcode = 0xb3cd; // LGDR
- put(sink, &enc_rre(opcode, rd.to_reg(), rn));
- } else {
- let (opcode, m4) = (0xe721, 3); // VLVG
- put(sink, &enc_vrs_c(opcode, rd.to_reg(), zero_reg(), 0, rn, m4));
- }
- }
&Inst::LoadFpuConst32 { rd, const_data } => {
let rd = allocs.next_writable(rd);
@@ -2213,9 +2274,11 @@ impl MachInstEmit for Inst {
let reg = writable_spilltmp_reg().to_reg();
put(sink, &enc_ri_b(opcode, reg, 8));
sink.put4(const_data.swap_bytes());
- let inst = Inst::FpuLoad32 {
+ let inst = Inst::VecLoadLaneUndef {
+ size: 32,
rd,
mem: MemArg::reg(reg, MemFlags::trusted()),
+ lane_imm: 0,
};
inst.emit(&[], sink, emit_info, state);
}
@@ -2226,9 +2289,11 @@ impl MachInstEmit for Inst {
let reg = writable_spilltmp_reg().to_reg();
put(sink, &enc_ri_b(opcode, reg, 12));
sink.put8(const_data.swap_bytes());
- let inst = Inst::FpuLoad64 {
+ let inst = Inst::VecLoadLaneUndef {
+ size: 64,
rd,
mem: MemArg::reg(reg, MemFlags::trusted()),
+ lane_imm: 0,
};
inst.emit(&[], sink, emit_info, state);
}
@@ -2236,21 +2301,30 @@ impl MachInstEmit for Inst {
let rd = allocs.next_writable(rd);
let rn = allocs.next(rn);
- let (opcode, m3, m5, opcode_fpr) = match fpu_op {
- FPUOp1::Abs32 => (0xe7cc, 2, 2, 0xb300), // VFPSO, LPEBR
- FPUOp1::Abs64 => (0xe7cc, 3, 2, 0xb310), // VFPSO, LPDBR
- FPUOp1::Neg32 => (0xe7cc, 2, 0, 0xb303), // VFPSO, LCEBR
- FPUOp1::Neg64 => (0xe7cc, 3, 0, 0xb313), // VFPSO, LCDBR
- FPUOp1::NegAbs32 => (0xe7cc, 2, 1, 0xb301), // VFPSO, LNEBR
- FPUOp1::NegAbs64 => (0xe7cc, 3, 1, 0xb311), // VFPSO, LNDBR
- FPUOp1::Sqrt32 => (0xe7ce, 2, 0, 0xb314), // VFSQ, SQEBR
- FPUOp1::Sqrt64 => (0xe7ce, 3, 0, 0xb315), // VFSQ, SQDBR
- FPUOp1::Cvt32To64 => (0xe7c4, 2, 0, 0xb304), // VFLL, LDEBR
+ let (opcode, m3, m4, m5, opcode_fpr) = match fpu_op {
+ FPUOp1::Abs32 => (0xe7cc, 2, 8, 2, Some(0xb300)), // WFPSO, LPEBR
+ FPUOp1::Abs64 => (0xe7cc, 3, 8, 2, Some(0xb310)), // WFPSO, LPDBR
+ FPUOp1::Abs32x4 => (0xe7cc, 2, 0, 2, None), // VFPSO
+ FPUOp1::Abs64x2 => (0xe7cc, 3, 0, 2, None), // VFPSO
+ FPUOp1::Neg32 => (0xe7cc, 2, 8, 0, Some(0xb303)), // WFPSO, LCEBR
+ FPUOp1::Neg64 => (0xe7cc, 3, 8, 0, Some(0xb313)), // WFPSO, LCDBR
+ FPUOp1::Neg32x4 => (0xe7cc, 2, 0, 0, None), // VFPSO
+ FPUOp1::Neg64x2 => (0xe7cc, 3, 0, 0, None), // VFPSO
+ FPUOp1::NegAbs32 => (0xe7cc, 2, 8, 1, Some(0xb301)), // WFPSO, LNEBR
+ FPUOp1::NegAbs64 => (0xe7cc, 3, 8, 1, Some(0xb311)), // WFPSO, LNDBR
+ FPUOp1::NegAbs32x4 => (0xe7cc, 2, 0, 1, None), // VFPSO
+ FPUOp1::NegAbs64x2 => (0xe7cc, 3, 0, 1, None), // VFPSO
+ FPUOp1::Sqrt32 => (0xe7ce, 2, 8, 0, Some(0xb314)), // WFSQ, SQEBR
+ FPUOp1::Sqrt64 => (0xe7ce, 3, 8, 0, Some(0xb315)), // WFSQ, SQDBR
+ FPUOp1::Sqrt32x4 => (0xe7ce, 2, 0, 0, None), // VFSQ
+ FPUOp1::Sqrt64x2 => (0xe7ce, 3, 0, 0, None), // VFSQ
+ FPUOp1::Cvt32To64 => (0xe7c4, 2, 8, 0, Some(0xb304)), // WFLL, LDEBR
+ FPUOp1::Cvt32x4To64x2 => (0xe7c4, 2, 0, 0, None), // VFLL
};
- if is_fpr(rd.to_reg()) && is_fpr(rn) {
- put(sink, &enc_rre(opcode_fpr, rd.to_reg(), rn));
+ if m4 == 8 && is_fpr(rd.to_reg()) && is_fpr(rn) {
+ put(sink, &enc_rre(opcode_fpr.unwrap(), rd.to_reg(), rn));
} else {
- put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, m3, 8, m5));
+ put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, m3, m4, m5));
}
}
&Inst::FpuRRR { fpu_op, rd, rn, rm } => {
@@ -2258,24 +2332,45 @@ impl MachInstEmit for Inst {
let rn = allocs.next(rn);
let rm = allocs.next(rm);
- let (opcode, m4, m6, opcode_fpr) = match fpu_op {
- FPUOp2::Add32 => (0xe7e3, 2, 0, Some(0xb30a)), // VFA, AEBR
- FPUOp2::Add64 => (0xe7e3, 3, 0, Some(0xb31a)), // VFA, ADBR
- FPUOp2::Sub32 => (0xe7e2, 2, 0, Some(0xb30b)), // VFS, SEBR
- FPUOp2::Sub64 => (0xe7e2, 3, 0, Some(0xb31b)), // VFS, SDBR
- FPUOp2::Mul32 => (0xe7e7, 2, 0, Some(0xb317)), // VFM, MEEBR
- FPUOp2::Mul64 => (0xe7e7, 3, 0, Some(0xb31c)), // VFM, MDBR
- FPUOp2::Div32 => (0xe7e5, 2, 0, Some(0xb30d)), // VFD, DEBR
- FPUOp2::Div64 => (0xe7e5, 3, 0, Some(0xb31d)), // VFD, DDBR
- FPUOp2::Max32 => (0xe7ef, 2, 1, None), // VFMAX
- FPUOp2::Max64 => (0xe7ef, 3, 1, None), // VFMAX
- FPUOp2::Min32 => (0xe7ee, 2, 1, None), // VFMIN
- FPUOp2::Min64 => (0xe7ee, 3, 1, None), // VFMIN
+ let (opcode, m4, m5, m6, opcode_fpr) = match fpu_op {
+ FPUOp2::Add32 => (0xe7e3, 2, 8, 0, Some(0xb30a)), // WFA, AEBR
+ FPUOp2::Add64 => (0xe7e3, 3, 8, 0, Some(0xb31a)), // WFA, ADBR
+ FPUOp2::Add32x4 => (0xe7e3, 2, 0, 0, None), // VFA
+ FPUOp2::Add64x2 => (0xe7e3, 3, 0, 0, None), // VFA
+ FPUOp2::Sub32 => (0xe7e2, 2, 8, 0, Some(0xb30b)), // WFS, SEBR
+ FPUOp2::Sub64 => (0xe7e2, 3, 8, 0, Some(0xb31b)), // WFS, SDBR
+ FPUOp2::Sub32x4 => (0xe7e2, 2, 0, 0, None), // VFS
+ FPUOp2::Sub64x2 => (0xe7e2, 3, 0, 0, None), // VFS
+ FPUOp2::Mul32 => (0xe7e7, 2, 8, 0, Some(0xb317)), // WFM, MEEBR
+ FPUOp2::Mul64 => (0xe7e7, 3, 8, 0, Some(0xb31c)), // WFM, MDBR
+ FPUOp2::Mul32x4 => (0xe7e7, 2, 0, 0, None), // VFM
+ FPUOp2::Mul64x2 => (0xe7e7, 3, 0, 0, None), // VFM
+ FPUOp2::Div32 => (0xe7e5, 2, 8, 0, Some(0xb30d)), // WFD, DEBR
+ FPUOp2::Div64 => (0xe7e5, 3, 8, 0, Some(0xb31d)), // WFD, DDBR
+ FPUOp2::Div32x4 => (0xe7e5, 2, 0, 0, None), // VFD
+ FPUOp2::Div64x2 => (0xe7e5, 3, 0, 0, None), // VFD
+ FPUOp2::Max32 => (0xe7ef, 2, 8, 1, None), // WFMAX
+ FPUOp2::Max64 => (0xe7ef, 3, 8, 1, None), // WFMAX
+ FPUOp2::Max32x4 => (0xe7ef, 2, 0, 1, None), // VFMAX
+ FPUOp2::Max64x2 => (0xe7ef, 3, 0, 1, None), // VFMAX
+ FPUOp2::Min32 => (0xe7ee, 2, 8, 1, None), // WFMIN
+ FPUOp2::Min64 => (0xe7ee, 3, 8, 1, None), // WFMIN
+ FPUOp2::Min32x4 => (0xe7ee, 2, 0, 1, None), // VFMIN
+ FPUOp2::Min64x2 => (0xe7ee, 3, 0, 1, None), // VFMIN
+ FPUOp2::MaxPseudo32 => (0xe7ef, 2, 8, 3, None), // WFMAX
+ FPUOp2::MaxPseudo64 => (0xe7ef, 3, 8, 3, None), // WFMAX
+ FPUOp2::MaxPseudo32x4 => (0xe7ef, 2, 0, 3, None), // VFMAX
+ FPUOp2::MaxPseudo64x2 => (0xe7ef, 3, 0, 3, None), // VFMAX
+ FPUOp2::MinPseudo32 => (0xe7ee, 2, 8, 3, None), // WFMIN
+ FPUOp2::MinPseudo64 => (0xe7ee, 3, 8, 3, None), // WFMIN
+ FPUOp2::MinPseudo32x4 => (0xe7ee, 2, 0, 3, None), // VFMIN
+ FPUOp2::MinPseudo64x2 => (0xe7ee, 3, 0, 3, None), // VFMIN
};
- if opcode_fpr.is_some() && rd.to_reg() == rn && is_fpr(rn) && is_fpr(rm) {
+ if m5 == 8 && opcode_fpr.is_some() && rd.to_reg() == rn && is_fpr(rn) && is_fpr(rm)
+ {
put(sink, &enc_rre(opcode_fpr.unwrap(), rd.to_reg(), rm));
} else {
- put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, 8, m6));
+ put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, m5, m6));
}
}
&Inst::FpuRRRR {
@@ -2290,16 +2385,20 @@ impl MachInstEmit for Inst {
let rm = allocs.next(rm);
let ra = allocs.next(ra);
- let (opcode, m6, opcode_fpr) = match fpu_op {
- FPUOp3::MAdd32 => (0xe78f, 2, 0xb30e), // VFMA, MAEBR
- FPUOp3::MAdd64 => (0xe78f, 3, 0xb31e), // VFMA, MADBR
- FPUOp3::MSub32 => (0xe78e, 2, 0xb30f), // VFMS, MSEBR
- FPUOp3::MSub64 => (0xe78e, 3, 0xb31f), // VFMS, MSDBR
+ let (opcode, m5, m6, opcode_fpr) = match fpu_op {
+ FPUOp3::MAdd32 => (0xe78f, 8, 2, Some(0xb30e)), // WFMA, MAEBR
+ FPUOp3::MAdd64 => (0xe78f, 8, 3, Some(0xb31e)), // WFMA, MADBR
+ FPUOp3::MAdd32x4 => (0xe78f, 0, 2, None), // VFMA
+ FPUOp3::MAdd64x2 => (0xe78f, 0, 3, None), // VFMA
+ FPUOp3::MSub32 => (0xe78e, 8, 2, Some(0xb30f)), // WFMS, MSEBR
+ FPUOp3::MSub64 => (0xe78e, 8, 3, Some(0xb31f)), // WFMS, MSDBR
+ FPUOp3::MSub32x4 => (0xe78e, 0, 2, None), // VFMS
+ FPUOp3::MSub64x2 => (0xe78e, 0, 3, None), // VFMS
};
- if rd.to_reg() == ra && is_fpr(rn) && is_fpr(rm) && is_fpr(ra) {
- put(sink, &enc_rrd(opcode_fpr, rd.to_reg(), rm, rn));
+ if m5 == 8 && rd.to_reg() == ra && is_fpr(rn) && is_fpr(rm) && is_fpr(ra) {
+ put(sink, &enc_rrd(opcode_fpr.unwrap(), rd.to_reg(), rm, rn));
} else {
- put(sink, &enc_vrr_e(opcode, rd.to_reg(), rn, rm, ra, 8, m6));
+ put(sink, &enc_vrr_e(opcode, rd.to_reg(), rn, rm, ra, m5, m6));
}
}
&Inst::FpuRound { op, mode, rd, rn } => {
@@ -2315,26 +2414,37 @@ impl MachInstEmit for Inst {
FpuRoundMode::ToPosInfinity => 6,
FpuRoundMode::ToNegInfinity => 7,
};
- let (opcode, m3, opcode_fpr) = match op {
- FpuRoundOp::Cvt64To32 => (0xe7c5, 3, Some(0xb344)), // VFLR, LEDBR(A)
- FpuRoundOp::Round32 => (0xe7c7, 2, Some(0xb357)), // VFI, FIEBR
- FpuRoundOp::Round64 => (0xe7c7, 3, Some(0xb35f)), // VFI, FIDBR
- FpuRoundOp::ToSInt32 => (0xe7c2, 2, None), // VCSFP
- FpuRoundOp::ToSInt64 => (0xe7c2, 3, None), // VCSFP
- FpuRoundOp::ToUInt32 => (0xe7c0, 2, None), // VCLFP
- FpuRoundOp::ToUInt64 => (0xe7c0, 3, None), // VCLFP
- FpuRoundOp::FromSInt32 => (0xe7c3, 2, None), // VCFPS
- FpuRoundOp::FromSInt64 => (0xe7c3, 3, None), // VCFPS
- FpuRoundOp::FromUInt32 => (0xe7c1, 2, None), // VCFPL
- FpuRoundOp::FromUInt64 => (0xe7c1, 3, None), // VCFPL
+ let (opcode, m3, m4, opcode_fpr) = match op {
+ FpuRoundOp::Cvt64To32 => (0xe7c5, 3, 8, Some(0xb344)), // WFLR, LEDBR(A)
+ FpuRoundOp::Cvt64x2To32x4 => (0xe7c5, 3, 0, None), // VFLR
+ FpuRoundOp::Round32 => (0xe7c7, 2, 8, Some(0xb357)), // WFI, FIEBR
+ FpuRoundOp::Round64 => (0xe7c7, 3, 8, Some(0xb35f)), // WFI, FIDBR
+ FpuRoundOp::Round32x4 => (0xe7c7, 2, 0, None), // VFI
+ FpuRoundOp::Round64x2 => (0xe7c7, 3, 0, None), // VFI
+ FpuRoundOp::ToSInt32 => (0xe7c2, 2, 8, None), // WCSFP
+ FpuRoundOp::ToSInt64 => (0xe7c2, 3, 8, None), // WCSFP
+ FpuRoundOp::ToUInt32 => (0xe7c0, 2, 8, None), // WCLFP
+ FpuRoundOp::ToUInt64 => (0xe7c0, 3, 8, None), // WCLFP
+ FpuRoundOp::ToSInt32x4 => (0xe7c2, 2, 0, None), // VCSFP
+ FpuRoundOp::ToSInt64x2 => (0xe7c2, 3, 0, None), // VCSFP
+ FpuRoundOp::ToUInt32x4 => (0xe7c0, 2, 0, None), // VCLFP
+ FpuRoundOp::ToUInt64x2 => (0xe7c0, 3, 0, None), // VCLFP
+ FpuRoundOp::FromSInt32 => (0xe7c3, 2, 8, None), // WCFPS
+ FpuRoundOp::FromSInt64 => (0xe7c3, 3, 8, None), // WCFPS
+ FpuRoundOp::FromUInt32 => (0xe7c1, 2, 8, None), // WCFPL
+ FpuRoundOp::FromUInt64 => (0xe7c1, 3, 8, None), // WCFPL
+ FpuRoundOp::FromSInt32x4 => (0xe7c3, 2, 0, None), // VCFPS
+ FpuRoundOp::FromSInt64x2 => (0xe7c3, 3, 0, None), // VCFPS
+ FpuRoundOp::FromUInt32x4 => (0xe7c1, 2, 0, None), // VCFPL
+ FpuRoundOp::FromUInt64x2 => (0xe7c1, 3, 0, None), // VCFPL
};
- if opcode_fpr.is_some() && is_fpr(rd.to_reg()) && is_fpr(rn) {
+ if m4 == 8 && opcode_fpr.is_some() && is_fpr(rd.to_reg()) && is_fpr(rn) {
put(
sink,
&enc_rrf_cde(opcode_fpr.unwrap(), rd.to_reg(), rn, mode, 0),
);
} else {
- put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, m3, 8, mode));
+ put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, m3, m4, mode));
}
}
&Inst::FpuCmp32 { rn, rm } => {
@@ -2361,6 +2471,169 @@ impl MachInstEmit for Inst {
put(sink, &enc_vrr_a(opcode, rn, rm, 3, 0, 0));
}
}
+
+ &Inst::VecRRR { op, rd, rn, rm } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let rm = allocs.next(rm);
+
+ let (opcode, m4) = match op {
+ VecBinaryOp::Add8x16 => (0xe7f3, 0), // VAB
+ VecBinaryOp::Add16x8 => (0xe7f3, 1), // VAH
+ VecBinaryOp::Add32x4 => (0xe7f3, 2), // VAF
+ VecBinaryOp::Add64x2 => (0xe7f3, 3), // VAG
+ VecBinaryOp::Sub8x16 => (0xe7f7, 0), // VSB
+ VecBinaryOp::Sub16x8 => (0xe7f7, 1), // VSH
+ VecBinaryOp::Sub32x4 => (0xe7f7, 2), // VSF
+ VecBinaryOp::Sub64x2 => (0xe7f7, 3), // VSG
+ VecBinaryOp::Mul8x16 => (0xe7a2, 0), // VMLB
+ VecBinaryOp::Mul16x8 => (0xe7a2, 1), // VMLHW
+ VecBinaryOp::Mul32x4 => (0xe7a2, 2), // VMLF
+ VecBinaryOp::UMulHi8x16 => (0xe7a1, 0), // VMLHB
+ VecBinaryOp::UMulHi16x8 => (0xe7a1, 1), // VMLHH
+ VecBinaryOp::UMulHi32x4 => (0xe7a1, 2), // VMLHF
+ VecBinaryOp::SMulHi8x16 => (0xe7a3, 0), // VMHB
+ VecBinaryOp::SMulHi16x8 => (0xe7a3, 1), // VMHH
+ VecBinaryOp::SMulHi32x4 => (0xe7a3, 2), // VMHF
+ VecBinaryOp::UMulEven8x16 => (0xe7a4, 0), // VMLEB
+ VecBinaryOp::UMulEven16x8 => (0xe7a4, 1), // VMLEH
+ VecBinaryOp::UMulEven32x4 => (0xe7a4, 2), // VMLEF
+ VecBinaryOp::SMulEven8x16 => (0xe7a6, 0), // VMEB
+ VecBinaryOp::SMulEven16x8 => (0xe7a6, 1), // VMEH
+ VecBinaryOp::SMulEven32x4 => (0xe7a6, 2), // VMEF
+ VecBinaryOp::UMulOdd8x16 => (0xe7a5, 0), // VMLOB
+ VecBinaryOp::UMulOdd16x8 => (0xe7a5, 1), // VMLOH
+ VecBinaryOp::UMulOdd32x4 => (0xe7a5, 2), // VMLOF
+ VecBinaryOp::SMulOdd8x16 => (0xe7a7, 0), // VMOB
+ VecBinaryOp::SMulOdd16x8 => (0xe7a7, 1), // VMOH
+ VecBinaryOp::SMulOdd32x4 => (0xe7a7, 2), // VMOF
+ VecBinaryOp::UMax8x16 => (0xe7fd, 0), // VMXLB
+ VecBinaryOp::UMax16x8 => (0xe7fd, 1), // VMXLH
+ VecBinaryOp::UMax32x4 => (0xe7fd, 2), // VMXLF
+ VecBinaryOp::UMax64x2 => (0xe7fd, 3), // VMXLG
+ VecBinaryOp::SMax8x16 => (0xe7ff, 0), // VMXB
+ VecBinaryOp::SMax16x8 => (0xe7ff, 1), // VMXH
+ VecBinaryOp::SMax32x4 => (0xe7ff, 2), // VMXF
+ VecBinaryOp::SMax64x2 => (0xe7ff, 3), // VMXG
+ VecBinaryOp::UMin8x16 => (0xe7fc, 0), // VMNLB
+ VecBinaryOp::UMin16x8 => (0xe7fc, 1), // VMNLH
+ VecBinaryOp::UMin32x4 => (0xe7fc, 2), // VMNLF
+ VecBinaryOp::UMin64x2 => (0xe7fc, 3), // VMNLG
+ VecBinaryOp::SMin8x16 => (0xe7fe, 0), // VMNB
+ VecBinaryOp::SMin16x8 => (0xe7fe, 1), // VMNH
+ VecBinaryOp::SMin32x4 => (0xe7fe, 2), // VMNF
+ VecBinaryOp::SMin64x2 => (0xe7fe, 3), // VMNG
+ VecBinaryOp::UAvg8x16 => (0xe7f0, 0), // VAVGLB
+ VecBinaryOp::UAvg16x8 => (0xe7f0, 1), // VAVGLH
+ VecBinaryOp::UAvg32x4 => (0xe7f0, 2), // VAVGLF
+ VecBinaryOp::UAvg64x2 => (0xe7f0, 3), // VAVGLG
+ VecBinaryOp::SAvg8x16 => (0xe7f2, 0), // VAVGB
+ VecBinaryOp::SAvg16x8 => (0xe7f2, 1), // VAVGH
+ VecBinaryOp::SAvg32x4 => (0xe7f2, 2), // VAVGF
+ VecBinaryOp::SAvg64x2 => (0xe7f2, 3), // VAVGG
+ VecBinaryOp::And128 => (0xe768, 0), // VN
+ VecBinaryOp::Orr128 => (0xe76a, 0), // VO
+ VecBinaryOp::Xor128 => (0xe76d, 0), // VX
+ VecBinaryOp::NotAnd128 => (0xe76e, 0), // VNN
+ VecBinaryOp::NotOrr128 => (0xe76b, 0), // VNO
+ VecBinaryOp::NotXor128 => (0xe76c, 0), // VNX
+ VecBinaryOp::AndNot128 => (0xe769, 0), // VNC
+ VecBinaryOp::OrrNot128 => (0xe76f, 0), // VOC
+ VecBinaryOp::BitPermute128 => (0xe785, 0), // VBPERM
+ VecBinaryOp::LShLByByte128 => (0xe775, 0), // VSLB
+ VecBinaryOp::LShRByByte128 => (0xe77d, 0), // VSRLB
+ VecBinaryOp::AShRByByte128 => (0xe77f, 0), // VSRAB
+ VecBinaryOp::LShLByBit128 => (0xe774, 0), // VSL
+ VecBinaryOp::LShRByBit128 => (0xe77c, 0), // VSRL
+ VecBinaryOp::AShRByBit128 => (0xe77e, 0), // VSRA
+ VecBinaryOp::Pack16x8 => (0xe794, 1), // VPKH
+ VecBinaryOp::Pack32x4 => (0xe794, 2), // VPKF
+ VecBinaryOp::Pack64x2 => (0xe794, 3), // VPKG
+ VecBinaryOp::PackUSat16x8 => (0xe795, 1), // VPKLSH
+ VecBinaryOp::PackUSat32x4 => (0xe795, 2), // VPKLSF
+ VecBinaryOp::PackUSat64x2 => (0xe795, 3), // VPKLSG
+ VecBinaryOp::PackSSat16x8 => (0xe797, 1), // VPKSH
+ VecBinaryOp::PackSSat32x4 => (0xe797, 2), // VPKSF
+ VecBinaryOp::PackSSat64x2 => (0xe797, 3), // VPKSG
+ VecBinaryOp::MergeLow8x16 => (0xe760, 0), // VMRLB
+ VecBinaryOp::MergeLow16x8 => (0xe760, 1), // VMRLH
+ VecBinaryOp::MergeLow32x4 => (0xe760, 2), // VMRLF
+ VecBinaryOp::MergeLow64x2 => (0xe760, 3), // VMRLG
+ VecBinaryOp::MergeHigh8x16 => (0xe761, 0), // VMRHB
+ VecBinaryOp::MergeHigh16x8 => (0xe761, 1), // VMRHH
+ VecBinaryOp::MergeHigh32x4 => (0xe761, 2), // VMRHF
+ VecBinaryOp::MergeHigh64x2 => (0xe761, 3), // VMRHG
+ };
+
+ put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, 0, 0));
+ }
+ &Inst::VecRR { op, rd, rn } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+
+ let (opcode, m3) = match op {
+ VecUnaryOp::Abs8x16 => (0xe7df, 0), // VLPB
+ VecUnaryOp::Abs16x8 => (0xe7df, 1), // VLPH
+ VecUnaryOp::Abs32x4 => (0xe7df, 2), // VLPF
+ VecUnaryOp::Abs64x2 => (0xe7df, 3), // VLPG
+ VecUnaryOp::Neg8x16 => (0xe7de, 0), // VLCB
+ VecUnaryOp::Neg16x8 => (0xe7de, 1), // VLCH
+ VecUnaryOp::Neg32x4 => (0xe7de, 2), // VLCF
+ VecUnaryOp::Neg64x2 => (0xe7de, 3), // VLCG
+ VecUnaryOp::Popcnt8x16 => (0xe750, 0), // VPOPCTB
+ VecUnaryOp::Popcnt16x8 => (0xe750, 1), // VPOPCTH
+ VecUnaryOp::Popcnt32x4 => (0xe750, 2), // VPOPCTF
+ VecUnaryOp::Popcnt64x2 => (0xe750, 3), // VPOPCTG
+ VecUnaryOp::UnpackULow8x16 => (0xe7d4, 0), // VUPLLB
+ VecUnaryOp::UnpackULow16x8 => (0xe7d4, 1), // VUPLLH
+ VecUnaryOp::UnpackULow32x4 => (0xe7d4, 2), // VUPLLF
+ VecUnaryOp::UnpackUHigh8x16 => (0xe7d5, 0), // VUPLHB
+ VecUnaryOp::UnpackUHigh16x8 => (0xe7d5, 1), // VUPLHH
+ VecUnaryOp::UnpackUHigh32x4 => (0xe7d5, 2), // VUPLHF
+ VecUnaryOp::UnpackSLow8x16 => (0xe7d6, 0), // VUPLB
+ VecUnaryOp::UnpackSLow16x8 => (0xe7d6, 1), // VUPLH
+ VecUnaryOp::UnpackSLow32x4 => (0xe7d6, 2), // VUPLF
+ VecUnaryOp::UnpackSHigh8x16 => (0xe7d7, 0), // VUPHB
+ VecUnaryOp::UnpackSHigh16x8 => (0xe7d7, 1), // VUPHH
+ VecUnaryOp::UnpackSHigh32x4 => (0xe7d7, 2), // VUPHF
+ };
+
+ put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, m3, 0, 0));
+ }
+ &Inst::VecShiftRR {
+ shift_op,
+ rd,
+ rn,
+ shift_imm,
+ shift_reg,
+ } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let shift_reg = allocs.next(shift_reg);
+
+ let (opcode, m4) = match shift_op {
+ VecShiftOp::RotL8x16 => (0xe733, 0), // VERLLB
+ VecShiftOp::RotL16x8 => (0xe733, 1), // VERLLH
+ VecShiftOp::RotL32x4 => (0xe733, 2), // VERLLF
+ VecShiftOp::RotL64x2 => (0xe733, 3), // VERLLG
+ VecShiftOp::LShL8x16 => (0xe730, 0), // VESLB
+ VecShiftOp::LShL16x8 => (0xe730, 1), // VESLH
+ VecShiftOp::LShL32x4 => (0xe730, 2), // VESLF
+ VecShiftOp::LShL64x2 => (0xe730, 3), // VESLG
+ VecShiftOp::LShR8x16 => (0xe738, 0), // VESRLB
+ VecShiftOp::LShR16x8 => (0xe738, 1), // VESRLH
+ VecShiftOp::LShR32x4 => (0xe738, 2), // VESRLF
+ VecShiftOp::LShR64x2 => (0xe738, 3), // VESRLG
+ VecShiftOp::AShR8x16 => (0xe73a, 0), // VESRAB
+ VecShiftOp::AShR16x8 => (0xe73a, 1), // VESRAH
+ VecShiftOp::AShR32x4 => (0xe73a, 2), // VESRAF
+ VecShiftOp::AShR64x2 => (0xe73a, 3), // VESRAG
+ };
+ put(
+ sink,
+ &enc_vrs_a(opcode, rd.to_reg(), shift_reg, shift_imm.into(), rn, m4),
+ );
+ }
&Inst::VecSelect { rd, rn, rm, ra } => {
let rd = allocs.next_writable(rd);
let rn = allocs.next(rn);
@@ -2370,6 +2643,442 @@ impl MachInstEmit for Inst {
let opcode = 0xe78d; // VSEL
put(sink, &enc_vrr_e(opcode, rd.to_reg(), rn, rm, ra, 0, 0));
}
+ &Inst::VecPermute { rd, rn, rm, ra } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let rm = allocs.next(rm);
+ let ra = allocs.next(ra);
+
+ let opcode = 0xe78c; // VPERM
+ put(sink, &enc_vrr_e(opcode, rd.to_reg(), rn, rm, ra, 0, 0));
+ }
+ &Inst::VecPermuteDWImm {
+ rd,
+ rn,
+ rm,
+ idx1,
+ idx2,
+ } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let rm = allocs.next(rm);
+ let m4 = (idx1 & 1) * 4 + (idx2 & 1);
+
+ let opcode = 0xe784; // VPDI
+ put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, 0, 0));
+ }
+ &Inst::VecIntCmp { op, rd, rn, rm } | &Inst::VecIntCmpS { op, rd, rn, rm } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let rm = allocs.next(rm);
+
+ let (opcode, m4) = match op {
+ VecIntCmpOp::CmpEq8x16 => (0xe7f8, 0), // VCEQB
+ VecIntCmpOp::CmpEq16x8 => (0xe7f8, 1), // VCEQH
+ VecIntCmpOp::CmpEq32x4 => (0xe7f8, 2), // VCEQF
+ VecIntCmpOp::CmpEq64x2 => (0xe7f8, 3), // VCEQG
+ VecIntCmpOp::SCmpHi8x16 => (0xe7fb, 0), // VCHB
+ VecIntCmpOp::SCmpHi16x8 => (0xe7fb, 1), // VCHH
+ VecIntCmpOp::SCmpHi32x4 => (0xe7fb, 2), // VCHG
+ VecIntCmpOp::SCmpHi64x2 => (0xe7fb, 3), // VCHG
+ VecIntCmpOp::UCmpHi8x16 => (0xe7f9, 0), // VCHLB
+ VecIntCmpOp::UCmpHi16x8 => (0xe7f9, 1), // VCHLH
+ VecIntCmpOp::UCmpHi32x4 => (0xe7f9, 2), // VCHLG
+ VecIntCmpOp::UCmpHi64x2 => (0xe7f9, 3), // VCHLG
+ };
+ let m5 = match self {
+ &Inst::VecIntCmp { .. } => 0,
+ &Inst::VecIntCmpS { .. } => 1,
+ _ => unreachable!(),
+ };
+
+ put(sink, &enc_vrr_b(opcode, rd.to_reg(), rn, rm, m4, m5));
+ }
+ &Inst::VecFloatCmp { op, rd, rn, rm } | &Inst::VecFloatCmpS { op, rd, rn, rm } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let rm = allocs.next(rm);
+
+ let (opcode, m4) = match op {
+ VecFloatCmpOp::CmpEq32x4 => (0xe7e8, 2), // VFCESB
+ VecFloatCmpOp::CmpEq64x2 => (0xe7e8, 3), // VFCEDB
+ VecFloatCmpOp::CmpHi32x4 => (0xe7eb, 2), // VFCHSB
+ VecFloatCmpOp::CmpHi64x2 => (0xe7eb, 3), // VFCHDB
+ VecFloatCmpOp::CmpHiEq32x4 => (0xe7ea, 2), // VFCHESB
+ VecFloatCmpOp::CmpHiEq64x2 => (0xe7ea, 3), // VFCHEDB
+ };
+ let m6 = match self {
+ &Inst::VecFloatCmp { .. } => 0,
+ &Inst::VecFloatCmpS { .. } => 1,
+ _ => unreachable!(),
+ };
+
+ put(sink, &enc_vrr_c(opcode, rd.to_reg(), rn, rm, m4, 0, m6));
+ }
+
+ &Inst::VecLoad { rd, ref mem } | &Inst::VecLoadRev { rd, ref mem } => {
+ let rd = allocs.next_writable(rd);
+ let mem = mem.with_allocs(&mut allocs);
+
+ let (opcode, m3) = match self {
+ &Inst::VecLoad { .. } => (0xe706, 0), // VL
+ &Inst::VecLoadRev { .. } => (0xe606, 4), // VLBRQ
+ _ => unreachable!(),
+ };
+ mem_vrx_emit(rd.to_reg(), &mem, opcode, m3, true, sink, emit_info, state);
+ }
+ &Inst::VecStore { rd, ref mem } | &Inst::VecStoreRev { rd, ref mem } => {
+ let rd = allocs.next(rd);
+ let mem = mem.with_allocs(&mut allocs);
+
+ let (opcode, m3) = match self {
+ &Inst::VecStore { .. } => (0xe70e, 0), // VST
+ &Inst::VecStoreRev { .. } => (0xe60e, 4), // VSTBRQ
+ _ => unreachable!(),
+ };
+ mem_vrx_emit(rd, &mem, opcode, m3, true, sink, emit_info, state);
+ }
+ &Inst::VecLoadReplicate { size, rd, ref mem }
+ | &Inst::VecLoadReplicateRev { size, rd, ref mem } => {
+ let rd = allocs.next_writable(rd);
+ let mem = mem.with_allocs(&mut allocs);
+
+ let (opcode, m3) = match (self, size) {
+ (&Inst::VecLoadReplicate { .. }, 8) => (0xe705, 0), // VLREPB
+ (&Inst::VecLoadReplicate { .. }, 16) => (0xe705, 1), // VLREPH
+ (&Inst::VecLoadReplicate { .. }, 32) => (0xe705, 2), // VLREPF
+ (&Inst::VecLoadReplicate { .. }, 64) => (0xe705, 3), // VLREPG
+ (&Inst::VecLoadReplicateRev { .. }, 16) => (0xe605, 1), // VLREPBRH
+ (&Inst::VecLoadReplicateRev { .. }, 32) => (0xe605, 2), // VLREPBRF
+ (&Inst::VecLoadReplicateRev { .. }, 64) => (0xe605, 3), // VLREPBRG
+ _ => unreachable!(),
+ };
+ mem_vrx_emit(rd.to_reg(), &mem, opcode, m3, true, sink, emit_info, state);
+ }
+
+ &Inst::VecMov { rd, rn } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+
+ let opcode = 0xe756; // VLR
+ put(sink, &enc_vrr_a(opcode, rd.to_reg(), rn, 0, 0, 0));
+ }
+ &Inst::VecCMov { rd, cond, rm } => {
+ let rd = allocs.next_writable(rd);
+ let rm = allocs.next(rm);
+
+ let opcode = 0xa74; // BCR
+ put(sink, &enc_ri_c(opcode, cond.invert().bits(), 4 + 6));
+ let opcode = 0xe756; // VLR
+ put(sink, &enc_vrr_a(opcode, rd.to_reg(), rm, 0, 0, 0));
+ }
+ &Inst::MovToVec128 { rd, rn, rm } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let rm = allocs.next(rm);
+
+ let opcode = 0xe762; // VLVGP
+ put(sink, &enc_vrr_f(opcode, rd.to_reg(), rn, rm));
+ }
+ &Inst::VecLoadConst { rd, const_data } => {
+ let rd = allocs.next_writable(rd);
+
+ let opcode = 0xa75; // BRAS
+ let reg = writable_spilltmp_reg().to_reg();
+ put(sink, &enc_ri_b(opcode, reg, 20));
+ for i in const_data.to_be_bytes().iter() {
+ sink.put1(*i);
+ }
+ let inst = Inst::VecLoad {
+ rd,
+ mem: MemArg::reg(reg, MemFlags::trusted()),
+ };
+ inst.emit(&[], sink, emit_info, state);
+ }
+ &Inst::VecLoadConstReplicate {
+ size,
+ rd,
+ const_data,
+ } => {
+ let rd = allocs.next_writable(rd);
+
+ let opcode = 0xa75; // BRAS
+ let reg = writable_spilltmp_reg().to_reg();
+ put(sink, &enc_ri_b(opcode, reg, (4 + size / 8) as i32));
+ for i in 0..size / 8 {
+ sink.put1((const_data >> (size - 8 - 8 * i)) as u8);
+ }
+ let inst = Inst::VecLoadReplicate {
+ size,
+ rd,
+ mem: MemArg::reg(reg, MemFlags::trusted()),
+ };
+ inst.emit(&[], sink, emit_info, state);
+ }
+ &Inst::VecImmByteMask { rd, mask } => {
+ let rd = allocs.next_writable(rd);
+ let opcode = 0xe744; // VGBM
+ put(sink, &enc_vri_a(opcode, rd.to_reg(), mask, 0));
+ }
+ &Inst::VecImmBitMask {
+ size,
+ rd,
+ start_bit,
+ end_bit,
+ } => {
+ let rd = allocs.next_writable(rd);
+ let (opcode, m4) = match size {
+ 8 => (0xe746, 0), // VGMB
+ 16 => (0xe746, 1), // VGMH
+ 32 => (0xe746, 2), // VGMF
+ 64 => (0xe746, 3), // VGMG
+ _ => unreachable!(),
+ };
+ put(
+ sink,
+ &enc_vri_b(opcode, rd.to_reg(), start_bit, end_bit, m4),
+ );
+ }
+ &Inst::VecImmReplicate { size, rd, imm } => {
+ let rd = allocs.next_writable(rd);
+ let (opcode, m3) = match size {
+ 8 => (0xe745, 0), // VREPIB
+ 16 => (0xe745, 1), // VREPIH
+ 32 => (0xe745, 2), // VREPIF
+ 64 => (0xe745, 3), // VREPIG
+ _ => unreachable!(),
+ };
+ put(sink, &enc_vri_a(opcode, rd.to_reg(), imm as u16, m3));
+ }
+
+ &Inst::VecLoadLane {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ }
+ | &Inst::VecLoadLaneUndef {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ }
+ | &Inst::VecLoadLaneRev {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ }
+ | &Inst::VecLoadLaneRevUndef {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ } => {
+ let rd = allocs.next_writable(rd);
+ let mem = mem.with_allocs(&mut allocs);
+
+ let (opcode_vrx, opcode_rx, opcode_rxy) = match (self, size) {
+ (&Inst::VecLoadLane { .. }, 8) => (0xe700, None, None), // VLEB
+ (&Inst::VecLoadLane { .. }, 16) => (0xe701, None, None), // VLEH
+ (&Inst::VecLoadLane { .. }, 32) => (0xe703, None, None), // VLEF
+ (&Inst::VecLoadLane { .. }, 64) => (0xe702, None, None), // VLEG
+ (&Inst::VecLoadLaneUndef { .. }, 8) => (0xe700, None, None), // VLEB
+ (&Inst::VecLoadLaneUndef { .. }, 16) => (0xe701, None, None), // VLEH
+ (&Inst::VecLoadLaneUndef { .. }, 32) => (0xe703, Some(0x78), Some(0xed64)), // VLEF, LE(Y)
+ (&Inst::VecLoadLaneUndef { .. }, 64) => (0xe702, Some(0x68), Some(0xed65)), // VLEG, LD(Y)
+ (&Inst::VecLoadLaneRev { .. }, 16) => (0xe601, None, None), // VLEBRH
+ (&Inst::VecLoadLaneRev { .. }, 32) => (0xe603, None, None), // VLEBRF
+ (&Inst::VecLoadLaneRev { .. }, 64) => (0xe602, None, None), // VLEBRG
+ (&Inst::VecLoadLaneRevUndef { .. }, 16) => (0xe601, None, None), // VLEBRH
+ (&Inst::VecLoadLaneRevUndef { .. }, 32) => (0xe603, None, None), // VLEBRF
+ (&Inst::VecLoadLaneRevUndef { .. }, 64) => (0xe602, None, None), // VLEBRG
+ _ => unreachable!(),
+ };
+
+ let rd = rd.to_reg();
+ if lane_imm == 0 && is_fpr(rd) && opcode_rx.is_some() {
+ mem_emit(
+ rd, &mem, opcode_rx, opcode_rxy, None, true, sink, emit_info, state,
+ );
+ } else {
+ mem_vrx_emit(
+ rd,
+ &mem,
+ opcode_vrx,
+ lane_imm.into(),
+ true,
+ sink,
+ emit_info,
+ state,
+ );
+ }
+ }
+ &Inst::VecStoreLane {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ }
+ | &Inst::VecStoreLaneRev {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ } => {
+ let rd = allocs.next(rd);
+ let mem = mem.with_allocs(&mut allocs);
+
+ let (opcode_vrx, opcode_rx, opcode_rxy) = match (self, size) {
+ (&Inst::VecStoreLane { .. }, 8) => (0xe708, None, None), // VSTEB
+ (&Inst::VecStoreLane { .. }, 16) => (0xe709, None, None), // VSTEH
+ (&Inst::VecStoreLane { .. }, 32) => (0xe70b, Some(0x70), Some(0xed66)), // VSTEF, STE(Y)
+ (&Inst::VecStoreLane { .. }, 64) => (0xe70a, Some(0x60), Some(0xed67)), // VSTEG, STD(Y)
+ (&Inst::VecStoreLaneRev { .. }, 16) => (0xe609, None, None), // VSTEBRH
+ (&Inst::VecStoreLaneRev { .. }, 32) => (0xe60b, None, None), // VSTEBRF
+ (&Inst::VecStoreLaneRev { .. }, 64) => (0xe60a, None, None), // VSTEBRG
+ _ => unreachable!(),
+ };
+
+ if lane_imm == 0 && is_fpr(rd) && opcode_rx.is_some() {
+ mem_emit(
+ rd, &mem, opcode_rx, opcode_rxy, None, true, sink, emit_info, state,
+ );
+ } else {
+ mem_vrx_emit(
+ rd,
+ &mem,
+ opcode_vrx,
+ lane_imm.into(),
+ true,
+ sink,
+ emit_info,
+ state,
+ );
+ }
+ }
+ &Inst::VecInsertLane {
+ size,
+ rd,
+ rn,
+ lane_imm,
+ lane_reg,
+ } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let lane_reg = allocs.next(lane_reg);
+
+ let (opcode_vrs, m4) = match size {
+ 8 => (0xe722, 0), // VLVGB
+ 16 => (0xe722, 1), // VLVGH
+ 32 => (0xe722, 2), // VLVGF
+ 64 => (0xe722, 3), // VLVGG
+ _ => unreachable!(),
+ };
+ put(
+ sink,
+ &enc_vrs_b(opcode_vrs, rd.to_reg(), lane_reg, lane_imm.into(), rn, m4),
+ );
+ }
+ &Inst::VecInsertLaneUndef {
+ size,
+ rd,
+ rn,
+ lane_imm,
+ lane_reg,
+ } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let lane_reg = allocs.next(lane_reg);
+
+ let (opcode_vrs, m4, opcode_rre) = match size {
+ 8 => (0xe722, 0, None), // VLVGB
+ 16 => (0xe722, 1, None), // VLVGH
+ 32 => (0xe722, 2, None), // VLVGF
+ 64 => (0xe722, 3, Some(0xb3c1)), // VLVGG, LDGR
+ _ => unreachable!(),
+ };
+ if opcode_rre.is_some()
+ && lane_imm == 0
+ && lane_reg == zero_reg()
+ && is_fpr(rd.to_reg())
+ {
+ put(sink, &enc_rre(opcode_rre.unwrap(), rd.to_reg(), rn));
+ } else {
+ put(
+ sink,
+ &enc_vrs_b(opcode_vrs, rd.to_reg(), lane_reg, lane_imm.into(), rn, m4),
+ );
+ }
+ }
+ &Inst::VecExtractLane {
+ size,
+ rd,
+ rn,
+ lane_imm,
+ lane_reg,
+ } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+ let lane_reg = allocs.next(lane_reg);
+
+ let (opcode_vrs, m4, opcode_rre) = match size {
+ 8 => (0xe721, 0, None), // VLGVB
+ 16 => (0xe721, 1, None), // VLGVH
+ 32 => (0xe721, 2, None), // VLGVF
+ 64 => (0xe721, 3, Some(0xb3cd)), // VLGVG, LGDR
+ _ => unreachable!(),
+ };
+ if opcode_rre.is_some() && lane_imm == 0 && lane_reg == zero_reg() && is_fpr(rn) {
+ put(sink, &enc_rre(opcode_rre.unwrap(), rd.to_reg(), rn));
+ } else {
+ put(
+ sink,
+ &enc_vrs_c(opcode_vrs, rd.to_reg(), lane_reg, lane_imm.into(), rn, m4),
+ );
+ }
+ }
+ &Inst::VecInsertLaneImm {
+ size,
+ rd,
+ imm,
+ lane_imm,
+ } => {
+ let rd = allocs.next_writable(rd);
+
+ let opcode = match size {
+ 8 => 0xe740, // VLEIB
+ 16 => 0xe741, // LEIVH
+ 32 => 0xe743, // VLEIF
+ 64 => 0xe742, // VLEIG
+ _ => unreachable!(),
+ };
+ put(
+ sink,
+ &enc_vri_a(opcode, rd.to_reg(), imm as u16, lane_imm.into()),
+ );
+ }
+ &Inst::VecReplicateLane {
+ size,
+ rd,
+ rn,
+ lane_imm,
+ } => {
+ let rd = allocs.next_writable(rd);
+ let rn = allocs.next(rn);
+
+ let (opcode, m4) = match size {
+ 8 => (0xe74d, 0), // VREPB
+ 16 => (0xe74d, 1), // VREPH
+ 32 => (0xe74d, 2), // VREPF
+ 64 => (0xe74d, 3), // VREPG
+ _ => unreachable!(),
+ };
+ put(
+ sink,
+ &enc_vri_c(opcode, rd.to_reg(), lane_imm.into(), rn, m4),
+ );
+ }
&Inst::Call { link, ref info } => {
let link = allocs.next_writable(link);
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
index a398c798a958..5dd423801fad 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
@@ -6808,6 +6808,8 @@ fn test_s390x_binemit() {
defs: smallvec![],
clobbers: PRegSet::empty(),
opcode: Opcode::Call,
+ caller_callconv: CallConv::SystemV,
+ callee_callconv: CallConv::SystemV,
}),
},
"C0E500000000",
@@ -6823,6 +6825,8 @@ fn test_s390x_binemit() {
defs: smallvec![],
clobbers: PRegSet::empty(),
opcode: Opcode::CallIndirect,
+ caller_callconv: CallConv::SystemV,
+ callee_callconv: CallConv::SystemV,
}),
},
"0DE1",
@@ -6953,71 +6957,6 @@ fn test_s390x_binemit() {
"jno 10 ; vlr %v8, %v20",
));
- insns.push((
- Inst::MovToFpr64 {
- rd: writable_vr(8),
- rn: gpr(4),
- },
- "B3C10084",
- "ldgr %f8, %r4",
- ));
- insns.push((
- Inst::MovToFpr64 {
- rd: writable_vr(24),
- rn: gpr(4),
- },
- "E78400003822",
- "vlvgg %v24, %r4, 0",
- ));
- insns.push((
- Inst::MovToFpr32 {
- rd: writable_vr(8),
- rn: gpr(4),
- },
- "E78400002022",
- "vlvgf %v8, %r4, 0",
- ));
- insns.push((
- Inst::MovToFpr32 {
- rd: writable_vr(24),
- rn: gpr(4),
- },
- "E78400002822",
- "vlvgf %v24, %r4, 0",
- ));
- insns.push((
- Inst::MovFromFpr64 {
- rd: writable_gpr(8),
- rn: vr(4),
- },
- "B3CD0084",
- "lgdr %r8, %f4",
- ));
- insns.push((
- Inst::MovFromFpr64 {
- rd: writable_gpr(8),
- rn: vr(20),
- },
- "E78400003421",
- "vlgvg %r8, %v20, 0",
- ));
- insns.push((
- Inst::MovFromFpr32 {
- rd: writable_gpr(8),
- rn: vr(4),
- },
- "E78400002021",
- "vlgvf %r8, %v4, 0",
- ));
- insns.push((
- Inst::MovFromFpr32 {
- rd: writable_gpr(8),
- rn: vr(20),
- },
- "E78400002421",
- "vlgvf %r8, %v20, 0",
- ));
-
insns.push((
Inst::FpuRR {
fpu_op: FPUOp1::Abs32,
@@ -7036,6 +6975,15 @@ fn test_s390x_binemit() {
"E78C002828CC",
"wflpsb %v24, %f12",
));
+ insns.push((
+ Inst::FpuRR {
+ fpu_op: FPUOp1::Abs32x4,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C002028CC",
+ "vflpsb %v24, %v12",
+ ));
insns.push((
Inst::FpuRR {
fpu_op: FPUOp1::Abs64,
@@ -7054,6 +7002,15 @@ fn test_s390x_binemit() {
"E78C002838CC",
"wflpdb %v24, %f12",
));
+ insns.push((
+ Inst::FpuRR {
+ fpu_op: FPUOp1::Abs64x2,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C002038CC",
+ "vflpdb %v24, %v12",
+ ));
insns.push((
Inst::FpuRR {
fpu_op: FPUOp1::Neg32,
@@ -7072,6 +7029,15 @@ fn test_s390x_binemit() {
"E78C000828CC",
"wflcsb %v24, %f12",
));
+ insns.push((
+ Inst::FpuRR {
+ fpu_op: FPUOp1::Neg32x4,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C000028CC",
+ "vflcsb %v24, %v12",
+ ));
insns.push((
Inst::FpuRR {
fpu_op: FPUOp1::Neg64,
@@ -7090,6 +7056,15 @@ fn test_s390x_binemit() {
"E78C000838CC",
"wflcdb %v24, %f12",
));
+ insns.push((
+ Inst::FpuRR {
+ fpu_op: FPUOp1::Neg64x2,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C000038CC",
+ "vflcdb %v24, %v12",
+ ));
insns.push((
Inst::FpuRR {
fpu_op: FPUOp1::NegAbs32,
@@ -7108,6 +7083,15 @@ fn test_s390x_binemit() {
"E78C001828CC",
"wflnsb %v24, %f12",
));
+ insns.push((
+ Inst::FpuRR {
+ fpu_op: FPUOp1::NegAbs32x4,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001028CC",
+ "vflnsb %v24, %v12",
+ ));
insns.push((
Inst::FpuRR {
fpu_op: FPUOp1::NegAbs64,
@@ -7126,6 +7110,15 @@ fn test_s390x_binemit() {
"E78C001838CC",
"wflndb %v24, %f12",
));
+ insns.push((
+ Inst::FpuRR {
+ fpu_op: FPUOp1::NegAbs64x2,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001038CC",
+ "vflndb %v24, %v12",
+ ));
insns.push((
Inst::FpuRR {
fpu_op: FPUOp1::Sqrt32,
@@ -7144,6 +7137,15 @@ fn test_s390x_binemit() {
"E78C000828CE",
"wfsqsb %v24, %f12",
));
+ insns.push((
+ Inst::FpuRR {
+ fpu_op: FPUOp1::Sqrt32x4,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C000028CE",
+ "vfsqsb %v24, %v12",
+ ));
insns.push((
Inst::FpuRR {
fpu_op: FPUOp1::Sqrt64,
@@ -7162,6 +7164,15 @@ fn test_s390x_binemit() {
"E78C000838CE",
"wfsqdb %v24, %f12",
));
+ insns.push((
+ Inst::FpuRR {
+ fpu_op: FPUOp1::Sqrt64x2,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C000038CE",
+ "vfsqdb %v24, %v12",
+ ));
insns.push((
Inst::FpuRR {
fpu_op: FPUOp1::Cvt32To64,
@@ -7180,6 +7191,15 @@ fn test_s390x_binemit() {
"E78C000828C4",
"wldeb %v24, %f12",
));
+ insns.push((
+ Inst::FpuRR {
+ fpu_op: FPUOp1::Cvt32x4To64x2,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C000028C4",
+ "vldeb %v24, %v12",
+ ));
insns.push((
Inst::FpuRRR {
@@ -7201,6 +7221,16 @@ fn test_s390x_binemit() {
"E748C00828E3",
"wfasb %v20, %f8, %f12",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Add32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028E3",
+ "vfasb %v20, %v8, %v12",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Add64,
@@ -7221,6 +7251,16 @@ fn test_s390x_binemit() {
"E748C00838E3",
"wfadb %v20, %f8, %f12",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Add64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038E3",
+ "vfadb %v20, %v8, %v12",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Sub32,
@@ -7241,6 +7281,16 @@ fn test_s390x_binemit() {
"E748C00828E2",
"wfssb %v20, %f8, %f12",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Sub32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028E2",
+ "vfssb %v20, %v8, %v12",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Sub64,
@@ -7261,6 +7311,16 @@ fn test_s390x_binemit() {
"E748C00838E2",
"wfsdb %v20, %f8, %f12",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Sub64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038E2",
+ "vfsdb %v20, %v8, %v12",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Mul32,
@@ -7281,6 +7341,16 @@ fn test_s390x_binemit() {
"E748C00828E7",
"wfmsb %v20, %f8, %f12",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Mul32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028E7",
+ "vfmsb %v20, %v8, %v12",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Mul64,
@@ -7301,6 +7371,16 @@ fn test_s390x_binemit() {
"E748C00838E7",
"wfmdb %v20, %f8, %f12",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Mul64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038E7",
+ "vfmdb %v20, %v8, %v12",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Div32,
@@ -7321,6 +7401,16 @@ fn test_s390x_binemit() {
"E748C00828E5",
"wfdsb %v20, %f8, %f12",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Div32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028E5",
+ "vfdsb %v20, %v8, %v12",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Div64,
@@ -7341,6 +7431,16 @@ fn test_s390x_binemit() {
"E748C00838E5",
"wfddb %v20, %f8, %f12",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Div64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038E5",
+ "vfddb %v20, %v8, %v12",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Max32,
@@ -7351,6 +7451,16 @@ fn test_s390x_binemit() {
"E746801820EF",
"wfmaxsb %f4, %f6, %f8, 1",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Max32x4,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ },
+ "E746801020EF",
+ "vfmaxsb %v4, %v6, %v8, 1",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Max64,
@@ -7361,6 +7471,16 @@ fn test_s390x_binemit() {
"E746801832EF",
"wfmaxdb %f4, %f6, %v24, 1",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Max64x2,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(24),
+ },
+ "E746801032EF",
+ "vfmaxdb %v4, %v6, %v24, 1",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Min32,
@@ -7371,6 +7491,16 @@ fn test_s390x_binemit() {
"E746801820EE",
"wfminsb %f4, %f6, %f8, 1",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Min32x4,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ },
+ "E746801020EE",
+ "vfminsb %v4, %v6, %v8, 1",
+ ));
insns.push((
Inst::FpuRRR {
fpu_op: FPUOp2::Min64,
@@ -7381,6 +7511,96 @@ fn test_s390x_binemit() {
"E746801830EE",
"wfmindb %f4, %f6, %f8, 1",
));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::Min64x2,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ },
+ "E746801030EE",
+ "vfmindb %v4, %v6, %v8, 1",
+ ));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::MaxPseudo32,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ },
+ "E746803820EF",
+ "wfmaxsb %f4, %f6, %f8, 3",
+ ));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::MaxPseudo32x4,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ },
+ "E746803020EF",
+ "vfmaxsb %v4, %v6, %v8, 3",
+ ));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::MaxPseudo64,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(24),
+ },
+ "E746803832EF",
+ "wfmaxdb %f4, %f6, %v24, 3",
+ ));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::MaxPseudo64x2,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(24),
+ },
+ "E746803032EF",
+ "vfmaxdb %v4, %v6, %v24, 3",
+ ));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::MinPseudo32,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ },
+ "E746803820EE",
+ "wfminsb %f4, %f6, %f8, 3",
+ ));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::MinPseudo32x4,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ },
+ "E746803020EE",
+ "vfminsb %v4, %v6, %v8, 3",
+ ));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::MinPseudo64,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ },
+ "E746803830EE",
+ "wfmindb %f4, %f6, %f8, 3",
+ ));
+ insns.push((
+ Inst::FpuRRR {
+ fpu_op: FPUOp2::MinPseudo64x2,
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ },
+ "E746803030EE",
+ "vfmindb %v4, %v6, %v8, 3",
+ ));
insns.push((
Inst::FpuRRRR {
@@ -7404,6 +7624,17 @@ fn test_s390x_binemit() {
"E78CD208418F",
"wfmasb %f8, %f12, %f13, %v20",
));
+ insns.push((
+ Inst::FpuRRRR {
+ fpu_op: FPUOp3::MAdd32x4,
+ rd: writable_vr(8),
+ rn: vr(12),
+ rm: vr(13),
+ ra: vr(20),
+ },
+ "E78CD200418F",
+ "vfmasb %v8, %v12, %v13, %v20",
+ ));
insns.push((
Inst::FpuRRRR {
fpu_op: FPUOp3::MAdd64,
@@ -7428,14 +7659,25 @@ fn test_s390x_binemit() {
));
insns.push((
Inst::FpuRRRR {
- fpu_op: FPUOp3::MSub32,
+ fpu_op: FPUOp3::MAdd64x2,
rd: writable_vr(8),
rn: vr(12),
rm: vr(13),
- ra: vr(8),
+ ra: vr(20),
},
- "B30F80CD",
- "msebr %f8, %f12, %f13",
+ "E78CD300418F",
+ "vfmadb %v8, %v12, %v13, %v20",
+ ));
+ insns.push((
+ Inst::FpuRRRR {
+ fpu_op: FPUOp3::MSub32,
+ rd: writable_vr(8),
+ rn: vr(12),
+ rm: vr(13),
+ ra: vr(8),
+ },
+ "B30F80CD",
+ "msebr %f8, %f12, %f13",
));
insns.push((
Inst::FpuRRRR {
@@ -7448,6 +7690,17 @@ fn test_s390x_binemit() {
"E78CD208418E",
"wfmssb %f8, %f12, %f13, %v20",
));
+ insns.push((
+ Inst::FpuRRRR {
+ fpu_op: FPUOp3::MSub32x4,
+ rd: writable_vr(8),
+ rn: vr(12),
+ rm: vr(13),
+ ra: vr(20),
+ },
+ "E78CD200418E",
+ "vfmssb %v8, %v12, %v13, %v20",
+ ));
insns.push((
Inst::FpuRRRR {
fpu_op: FPUOp3::MSub64,
@@ -7470,6 +7723,17 @@ fn test_s390x_binemit() {
"E78CD308418E",
"wfmsdb %f8, %f12, %f13, %v20",
));
+ insns.push((
+ Inst::FpuRRRR {
+ fpu_op: FPUOp3::MSub64x2,
+ rd: writable_vr(8),
+ rn: vr(12),
+ rm: vr(13),
+ ra: vr(20),
+ },
+ "E78CD300418E",
+ "vfmsdb %v8, %v12, %v13, %v20",
+ ));
insns.push((
Inst::FpuCmp32 {
@@ -7505,202 +7769,2982 @@ fn test_s390x_binemit() {
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(1),
- mem: MemArg::BXD12 {
- base: gpr(2),
- index: zero_reg(),
- disp: UImm12::zero(),
- flags: MemFlags::trusted(),
- },
+ Inst::LoadFpuConst32 {
+ rd: writable_vr(8),
+ const_data: 1.0_f32.to_bits(),
},
- "78102000",
- "le %f1, 0(%r2)",
+ "A71500043F80000078801000",
+ "bras %r1, 8 ; data.f32 1 ; le %f8, 0(%r1)",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(1),
- mem: MemArg::BXD12 {
- base: gpr(2),
- index: zero_reg(),
- disp: UImm12::maybe_from_u64(4095).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::LoadFpuConst32 {
+ rd: writable_vr(24),
+ const_data: 1.0_f32.to_bits(),
},
- "78102FFF",
- "le %f1, 4095(%r2)",
+ "A71500043F800000E78010000803",
+ "bras %r1, 8 ; data.f32 1 ; vlef %v24, 0(%r1), 0",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(1),
- mem: MemArg::BXD20 {
- base: gpr(2),
- index: zero_reg(),
- disp: SImm20::maybe_from_i64(-524288).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::LoadFpuConst64 {
+ rd: writable_vr(8),
+ const_data: 1.0_f64.to_bits(),
},
- "ED1020008064",
- "ley %f1, -524288(%r2)",
+ "A71500063FF000000000000068801000",
+ "bras %r1, 12 ; data.f64 1 ; ld %f8, 0(%r1)",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(1),
- mem: MemArg::BXD20 {
- base: gpr(2),
- index: zero_reg(),
- disp: SImm20::maybe_from_i64(524287).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::LoadFpuConst64 {
+ rd: writable_vr(24),
+ const_data: 1.0_f64.to_bits(),
},
- "ED102FFF7F64",
- "ley %f1, 524287(%r2)",
+ "A71500063FF0000000000000E78010000802",
+ "bras %r1, 12 ; data.f64 1 ; vleg %v24, 0(%r1), 0",
));
+
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(17),
- mem: MemArg::BXD12 {
- base: gpr(2),
- index: zero_reg(),
- disp: UImm12::zero(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Cvt64To32,
+ mode: FpuRoundMode::Current,
+ rd: writable_vr(8),
+ rn: vr(12),
},
- "E71020000803",
- "vlef %v17, 0(%r2), 0",
+ "B344008C",
+ "ledbra %f8, %f12, 0",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(17),
- mem: MemArg::BXD12 {
- base: gpr(2),
- index: zero_reg(),
- disp: UImm12::maybe_from_u64(4095).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Cvt64To32,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
},
- "E7102FFF0803",
- "vlef %v17, 4095(%r2), 0",
+ "E78C001838C5",
+ "wledb %v24, %f12, 0, 1",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(1),
- mem: MemArg::BXD12 {
- base: gpr(3),
- index: gpr(2),
- disp: UImm12::zero(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Cvt64x2To32x4,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
},
- "78123000",
- "le %f1, 0(%r2,%r3)",
+ "E78C001038C5",
+ "vledb %v24, %v12, 0, 1",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(1),
- mem: MemArg::BXD12 {
- base: gpr(3),
- index: gpr(2),
- disp: UImm12::maybe_from_u64(4095).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Round32,
+ mode: FpuRoundMode::ToNegInfinity,
+ rd: writable_vr(8),
+ rn: vr(12),
},
- "78123FFF",
- "le %f1, 4095(%r2,%r3)",
+ "B357708C",
+ "fiebr %f8, %f12, 7",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(1),
- mem: MemArg::BXD20 {
- base: gpr(3),
- index: gpr(2),
- disp: SImm20::maybe_from_i64(-524288).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Round64,
+ mode: FpuRoundMode::ToNegInfinity,
+ rd: writable_vr(8),
+ rn: vr(12),
},
- "ED1230008064",
- "ley %f1, -524288(%r2,%r3)",
+ "B35F708C",
+ "fidbr %f8, %f12, 7",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(1),
- mem: MemArg::BXD20 {
- base: gpr(3),
- index: gpr(2),
- disp: SImm20::maybe_from_i64(524287).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Round32,
+ mode: FpuRoundMode::ToPosInfinity,
+ rd: writable_vr(8),
+ rn: vr(12),
},
- "ED123FFF7F64",
- "ley %f1, 524287(%r2,%r3)",
+ "B357608C",
+ "fiebr %f8, %f12, 6",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(17),
- mem: MemArg::BXD12 {
- base: gpr(3),
- index: gpr(2),
- disp: UImm12::zero(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Round64,
+ mode: FpuRoundMode::ToPosInfinity,
+ rd: writable_vr(8),
+ rn: vr(12),
},
- "E71230000803",
- "vlef %v17, 0(%r2,%r3), 0",
+ "B35F608C",
+ "fidbr %f8, %f12, 6",
));
insns.push((
- Inst::FpuLoad32 {
- rd: writable_vr(17),
- mem: MemArg::BXD12 {
- base: gpr(3),
- index: gpr(2),
- disp: UImm12::maybe_from_u64(4095).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Round32,
+ mode: FpuRoundMode::ToZero,
+ rd: writable_vr(8),
+ rn: vr(12),
},
- "E7123FFF0803",
- "vlef %v17, 4095(%r2,%r3), 0",
+ "B357508C",
+ "fiebr %f8, %f12, 5",
));
insns.push((
- Inst::FpuLoad64 {
- rd: writable_vr(1),
- mem: MemArg::BXD12 {
- base: gpr(2),
- index: zero_reg(),
- disp: UImm12::zero(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Round64,
+ mode: FpuRoundMode::ToZero,
+ rd: writable_vr(8),
+ rn: vr(12),
},
- "68102000",
- "ld %f1, 0(%r2)",
+ "B35F508C",
+ "fidbr %f8, %f12, 5",
));
insns.push((
- Inst::FpuLoad64 {
- rd: writable_vr(1),
- mem: MemArg::BXD12 {
- base: gpr(2),
- index: zero_reg(),
- disp: UImm12::maybe_from_u64(4095).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Round32,
+ mode: FpuRoundMode::ToNearestTiesToEven,
+ rd: writable_vr(8),
+ rn: vr(12),
},
- "68102FFF",
- "ld %f1, 4095(%r2)",
+ "B357408C",
+ "fiebr %f8, %f12, 4",
));
insns.push((
- Inst::FpuLoad64 {
- rd: writable_vr(1),
- mem: MemArg::BXD20 {
- base: gpr(2),
- index: zero_reg(),
- disp: SImm20::maybe_from_i64(-524288).unwrap(),
- flags: MemFlags::trusted(),
- },
+ Inst::FpuRound {
+ op: FpuRoundOp::Round64,
+ mode: FpuRoundMode::ToNearestTiesToEven,
+ rd: writable_vr(8),
+ rn: vr(12),
},
- "ED1020008065",
- "ldy %f1, -524288(%r2)",
- ));
+ "B35F408C",
+ "fidbr %f8, %f12, 4",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::Round32,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001828C7",
+ "wfisb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::Round32x4,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001028C7",
+ "vfisb %v24, %v12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::Round64,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001838C7",
+ "wfidb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::Round64x2,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001038C7",
+ "vfidb %v24, %v12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::ToSInt32,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001828C2",
+ "wcfeb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::ToSInt32x4,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001028C2",
+ "vcfeb %v24, %v12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::ToSInt64,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001838C2",
+ "wcgdb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::ToSInt64x2,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001038C2",
+ "vcgdb %v24, %v12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::ToUInt32,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001828C0",
+ "wclfeb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::ToUInt32x4,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001028C0",
+ "vclfeb %v24, %v12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::ToUInt64,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001838C0",
+ "wclgdb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::ToUInt64x2,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001038C0",
+ "vclgdb %v24, %v12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::FromSInt32,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001828C3",
+ "wcefb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::FromSInt32x4,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001028C3",
+ "vcefb %v24, %v12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::FromSInt64,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001838C3",
+ "wcdgb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::FromSInt64x2,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001038C3",
+ "vcdgb %v24, %v12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::FromUInt32,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001828C1",
+ "wcelfb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::FromUInt32x4,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001028C1",
+ "vcelfb %v24, %v12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::FromUInt64,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001838C1",
+ "wcdlgb %v24, %f12, 0, 1",
+ ));
+ insns.push((
+ Inst::FpuRound {
+ op: FpuRoundOp::FromUInt64x2,
+ mode: FpuRoundMode::ToNearest,
+ rd: writable_vr(24),
+ rn: vr(12),
+ },
+ "E78C001038C1",
+ "vcdlgb %v24, %v12, 0, 1",
+ ));
+
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Add8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008F3",
+ "vab %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Add16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018F3",
+ "vah %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Add32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028F3",
+ "vaf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Add64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038F3",
+ "vag %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Sub8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008F7",
+ "vsb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Sub16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018F7",
+ "vsh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Sub32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028F7",
+ "vsf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Sub64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038F7",
+ "vsg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Mul8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008A2",
+ "vmlb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Mul16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018A2",
+ "vmlhw %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Mul32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028A2",
+ "vmlf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMulHi8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008A1",
+ "vmlhb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMulHi16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018A1",
+ "vmlhh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMulHi32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028A1",
+ "vmlhf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMulHi8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008A3",
+ "vmhb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMulHi16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018A3",
+ "vmhh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMulHi32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028A3",
+ "vmhf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMulEven8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008A4",
+ "vmleb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMulEven16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018A4",
+ "vmleh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMulEven32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028A4",
+ "vmlef %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMulEven8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008A6",
+ "vmeb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMulEven16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018A6",
+ "vmeh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMulEven32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028A6",
+ "vmef %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMulOdd8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008A5",
+ "vmlob %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMulOdd16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018A5",
+ "vmloh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMulOdd32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028A5",
+ "vmlof %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMulOdd8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008A7",
+ "vmob %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMulOdd16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018A7",
+ "vmoh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMulOdd32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028A7",
+ "vmof %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMax8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008FD",
+ "vmxlb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMax16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018FD",
+ "vmxlh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMax32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028FD",
+ "vmxlf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMax64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038FD",
+ "vmxlg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMax8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008FF",
+ "vmxb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMax16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018FF",
+ "vmxh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMax32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028FF",
+ "vmxf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMax64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038FF",
+ "vmxg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMin8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008FC",
+ "vmnlb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMin16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018FC",
+ "vmnlh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMin32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028FC",
+ "vmnlf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UMin64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038FC",
+ "vmnlg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMin8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008FE",
+ "vmnb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMin16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018FE",
+ "vmnh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMin32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028FE",
+ "vmnf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SMin64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038FE",
+ "vmng %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UAvg8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008F0",
+ "vavglb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UAvg16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018F0",
+ "vavglh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UAvg32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028F0",
+ "vavglf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::UAvg64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038F0",
+ "vavglg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SAvg8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008F2",
+ "vavgb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SAvg16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018F2",
+ "vavgh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SAvg32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028F2",
+ "vavgf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::SAvg64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038F2",
+ "vavgg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::And128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0000868",
+ "vn %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Orr128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000086A",
+ "vo %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Xor128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000086D",
+ "vx %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::NotAnd128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000086E",
+ "vnn %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::NotOrr128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000086B",
+ "vno %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::NotXor128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000086C",
+ "vnx %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::AndNot128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0000869",
+ "vnc %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::OrrNot128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000086F",
+ "voc %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::BitPermute128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0000885",
+ "vbperm %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::LShLByByte128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0000875",
+ "vslb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::LShRByByte128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000087D",
+ "vsrlb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::AShRByByte128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000087F",
+ "vsrab %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::LShLByBit128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0000874",
+ "vsl %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::LShRByBit128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000087C",
+ "vsrl %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::AShRByBit128,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C000087E",
+ "vsra %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Pack16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0001894",
+ "vpkh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Pack32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0002894",
+ "vpkf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::Pack64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0003894",
+ "vpkg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::PackUSat16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0001895",
+ "vpklsh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::PackUSat32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0002895",
+ "vpklsf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::PackUSat64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0003895",
+ "vpklsg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::PackSSat16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0001897",
+ "vpksh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::PackSSat32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0002897",
+ "vpksf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::PackSSat64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0003897",
+ "vpksg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::MergeLow8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0000860",
+ "vmrlb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::MergeLow16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0001860",
+ "vmrlh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::MergeLow32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0002860",
+ "vmrlf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::MergeLow64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0003860",
+ "vmrlg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::MergeHigh8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0000861",
+ "vmrhb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::MergeHigh16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0001861",
+ "vmrhh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::MergeHigh32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0002861",
+ "vmrhf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecRRR {
+ op: VecBinaryOp::MergeHigh64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C0003861",
+ "vmrhg %v20, %v8, %v12",
+ ));
+
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Abs8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000008DF",
+ "vlpb %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Abs16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000018DF",
+ "vlph %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Abs32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000028DF",
+ "vlpf %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Abs64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000038DF",
+ "vlpg %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Neg8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000008DE",
+ "vlcb %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Neg16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000018DE",
+ "vlch %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Neg32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000028DE",
+ "vlcf %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Neg64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000038DE",
+ "vlcg %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Popcnt8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E74800000850",
+ "vpopctb %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Popcnt16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E74800001850",
+ "vpopcth %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Popcnt32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E74800002850",
+ "vpopctf %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::Popcnt64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E74800003850",
+ "vpopctg %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackULow8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000008D4",
+ "vupllb %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackULow16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000018D4",
+ "vupllh %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackULow32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000028D4",
+ "vupllf %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackUHigh8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000008D5",
+ "vuplhb %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackUHigh16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000018D5",
+ "vuplhh %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackUHigh32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000028D5",
+ "vuplhf %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackSLow8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000008D6",
+ "vuplb %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackSLow16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000018D6",
+ "vuplh %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackSLow32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000028D6",
+ "vuplf %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackSHigh8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000008D7",
+ "vuphb %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackSHigh16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000018D7",
+ "vuphh %v20, %v8",
+ ));
+ insns.push((
+ Inst::VecRR {
+ op: VecUnaryOp::UnpackSHigh32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ },
+ "E748000028D7",
+ "vuphf %v20, %v8",
+ ));
+
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::RotL8x16,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560030833",
+ "verllb %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::RotL16x8,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560031833",
+ "verllh %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::RotL32x4,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560032833",
+ "verllf %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::RotL64x2,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560033833",
+ "verllg %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::LShL8x16,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560030830",
+ "veslb %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::LShL16x8,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560031830",
+ "veslh %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::LShL32x4,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560032830",
+ "veslf %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::LShL64x2,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560033830",
+ "veslg %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::LShR8x16,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560030838",
+ "vesrlb %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::LShR16x8,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560031838",
+ "vesrlh %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::LShR32x4,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560032838",
+ "vesrlf %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::LShR64x2,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E74560033838",
+ "vesrlg %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::AShR8x16,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E7456003083A",
+ "vesrab %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::AShR16x8,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E7456003183A",
+ "vesrah %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::AShR32x4,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E7456003283A",
+ "vesraf %v20, %v5, 3(%r6)",
+ ));
+ insns.push((
+ Inst::VecShiftRR {
+ shift_op: VecShiftOp::AShR64x2,
+ rd: writable_vr(20),
+ rn: vr(5),
+ shift_imm: 3,
+ shift_reg: gpr(6),
+ },
+ "E7456003383A",
+ "vesrag %v20, %v5, 3(%r6)",
+ ));
+
+ insns.push((
+ Inst::VecSelect {
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ ra: vr(10),
+ },
+ "E7468000A08D",
+ "vsel %v4, %v6, %v8, %v10",
+ ));
+ insns.push((
+ Inst::VecSelect {
+ rd: writable_vr(20),
+ rn: vr(6),
+ rm: vr(8),
+ ra: vr(10),
+ },
+ "E7468000A88D",
+ "vsel %v20, %v6, %v8, %v10",
+ ));
+ insns.push((
+ Inst::VecSelect {
+ rd: writable_vr(4),
+ rn: vr(22),
+ rm: vr(8),
+ ra: vr(10),
+ },
+ "E7468000A48D",
+ "vsel %v4, %v22, %v8, %v10",
+ ));
+ insns.push((
+ Inst::VecSelect {
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(24),
+ ra: vr(10),
+ },
+ "E7468000A28D",
+ "vsel %v4, %v6, %v24, %v10",
+ ));
+ insns.push((
+ Inst::VecSelect {
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ ra: vr(26),
+ },
+ "E7468000A18D",
+ "vsel %v4, %v6, %v8, %v26",
+ ));
+ insns.push((
+ Inst::VecSelect {
+ rd: writable_vr(20),
+ rn: vr(22),
+ rm: vr(24),
+ ra: vr(26),
+ },
+ "E7468000AF8D",
+ "vsel %v20, %v22, %v24, %v26",
+ ));
+ insns.push((
+ Inst::VecPermute {
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ ra: vr(10),
+ },
+ "E7468000A08C",
+ "vperm %v4, %v6, %v8, %v10",
+ ));
+ insns.push((
+ Inst::VecPermute {
+ rd: writable_vr(20),
+ rn: vr(6),
+ rm: vr(8),
+ ra: vr(10),
+ },
+ "E7468000A88C",
+ "vperm %v20, %v6, %v8, %v10",
+ ));
+ insns.push((
+ Inst::VecPermute {
+ rd: writable_vr(4),
+ rn: vr(22),
+ rm: vr(8),
+ ra: vr(10),
+ },
+ "E7468000A48C",
+ "vperm %v4, %v22, %v8, %v10",
+ ));
+ insns.push((
+ Inst::VecPermute {
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(24),
+ ra: vr(10),
+ },
+ "E7468000A28C",
+ "vperm %v4, %v6, %v24, %v10",
+ ));
+ insns.push((
+ Inst::VecPermute {
+ rd: writable_vr(4),
+ rn: vr(6),
+ rm: vr(8),
+ ra: vr(26),
+ },
+ "E7468000A18C",
+ "vperm %v4, %v6, %v8, %v26",
+ ));
+ insns.push((
+ Inst::VecPermute {
+ rd: writable_vr(20),
+ rn: vr(22),
+ rm: vr(24),
+ ra: vr(26),
+ },
+ "E7468000AF8C",
+ "vperm %v20, %v22, %v24, %v26",
+ ));
+ insns.push((
+ Inst::VecPermuteDWImm {
+ rd: writable_vr(20),
+ rn: vr(6),
+ rm: vr(8),
+ idx1: 0,
+ idx2: 0,
+ },
+ "E74680000884",
+ "vpdi %v20, %v6, %v8, 0",
+ ));
+ insns.push((
+ Inst::VecPermuteDWImm {
+ rd: writable_vr(20),
+ rn: vr(6),
+ rm: vr(8),
+ idx1: 0,
+ idx2: 1,
+ },
+ "E74680001884",
+ "vpdi %v20, %v6, %v8, 1",
+ ));
+ insns.push((
+ Inst::VecPermuteDWImm {
+ rd: writable_vr(20),
+ rn: vr(6),
+ rm: vr(8),
+ idx1: 1,
+ idx2: 0,
+ },
+ "E74680004884",
+ "vpdi %v20, %v6, %v8, 4",
+ ));
+ insns.push((
+ Inst::VecPermuteDWImm {
+ rd: writable_vr(20),
+ rn: vr(6),
+ rm: vr(8),
+ idx1: 1,
+ idx2: 1,
+ },
+ "E74680005884",
+ "vpdi %v20, %v6, %v8, 5",
+ ));
+
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::CmpEq8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008F8",
+ "vceqb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::CmpEq16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018F8",
+ "vceqh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::CmpEq32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028F8",
+ "vceqf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::CmpEq64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038F8",
+ "vceqg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::SCmpHi8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008FB",
+ "vchb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::SCmpHi16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018FB",
+ "vchh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::SCmpHi32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028FB",
+ "vchf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::SCmpHi64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038FB",
+ "vchg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::UCmpHi8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00008F9",
+ "vchlb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::UCmpHi16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00018F9",
+ "vchlh %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::UCmpHi32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028F9",
+ "vchlf %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmp {
+ op: VecIntCmpOp::UCmpHi64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038F9",
+ "vchlg %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::CmpEq8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01008F8",
+ "vceqbs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::CmpEq16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01018F8",
+ "vceqhs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::CmpEq32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01028F8",
+ "vceqfs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::CmpEq64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01038F8",
+ "vceqgs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::SCmpHi8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01008FB",
+ "vchbs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::SCmpHi16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01018FB",
+ "vchhs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::SCmpHi32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01028FB",
+ "vchfs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::SCmpHi64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01038FB",
+ "vchgs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::UCmpHi8x16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01008F9",
+ "vchlbs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::UCmpHi16x8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01018F9",
+ "vchlhs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::UCmpHi32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01028F9",
+ "vchlfs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecIntCmpS {
+ op: VecIntCmpOp::UCmpHi64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01038F9",
+ "vchlgs %v20, %v8, %v12",
+ ));
+
+ insns.push((
+ Inst::VecFloatCmp {
+ op: VecFloatCmpOp::CmpEq32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028E8",
+ "vfcesb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmp {
+ op: VecFloatCmpOp::CmpEq64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038E8",
+ "vfcedb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmp {
+ op: VecFloatCmpOp::CmpHi32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028EB",
+ "vfchsb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmp {
+ op: VecFloatCmpOp::CmpHi64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038EB",
+ "vfchdb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmp {
+ op: VecFloatCmpOp::CmpHiEq32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00028EA",
+ "vfchesb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmp {
+ op: VecFloatCmpOp::CmpHiEq64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C00038EA",
+ "vfchedb %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmpS {
+ op: VecFloatCmpOp::CmpEq32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01028E8",
+ "vfcesbs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmpS {
+ op: VecFloatCmpOp::CmpEq64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01038E8",
+ "vfcedbs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmpS {
+ op: VecFloatCmpOp::CmpHi32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01028EB",
+ "vfchsbs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmpS {
+ op: VecFloatCmpOp::CmpHi64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01038EB",
+ "vfchdbs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmpS {
+ op: VecFloatCmpOp::CmpHiEq32x4,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01028EA",
+ "vfchesbs %v20, %v8, %v12",
+ ));
+ insns.push((
+ Inst::VecFloatCmpS {
+ op: VecFloatCmpOp::CmpHiEq64x2,
+ rd: writable_vr(20),
+ rn: vr(8),
+ rm: vr(12),
+ },
+ "E748C01038EA",
+ "vfchedbs %v20, %v8, %v12",
+ ));
+
+ insns.push((
+ Inst::VecLoad {
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E71020000806",
+ "vl %v17, 0(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoad {
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E7102FFF0806",
+ "vl %v17, 4095(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoad {
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E71230000806",
+ "vl %v17, 0(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecLoadRev {
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E61020004806",
+ "vlbrq %v17, 0(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadRev {
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E6102FFF4806",
+ "vlbrq %v17, 4095(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadRev {
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E61230004806",
+ "vlbrq %v17, 0(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecStore {
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E7102000080E",
+ "vst %v17, 0(%r2)",
+ ));
+ insns.push((
+ Inst::VecStore {
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E7102FFF080E",
+ "vst %v17, 4095(%r2)",
+ ));
+ insns.push((
+ Inst::VecStore {
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E7123000080E",
+ "vst %v17, 0(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecStoreRev {
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E6102000480E",
+ "vstbrq %v17, 0(%r2)",
+ ));
+ insns.push((
+ Inst::VecStoreRev {
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E6102FFF480E",
+ "vstbrq %v17, 4095(%r2)",
+ ));
+ insns.push((
+ Inst::VecStoreRev {
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E6123000480E",
+ "vstbrq %v17, 0(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecLoadReplicate {
+ size: 8,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(128).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E71020800805",
+ "vlrepb %v17, 128(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadReplicate {
+ size: 16,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(128).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E71020801805",
+ "vlreph %v17, 128(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadReplicate {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(128).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E71020802805",
+ "vlrepf %v17, 128(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadReplicate {
+ size: 64,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(128).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E71020803805",
+ "vlrepg %v17, 128(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadReplicateRev {
+ size: 16,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(128).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E61020801805",
+ "vlbrreph %v17, 128(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadReplicateRev {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(128).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E61020802805",
+ "vlbrrepf %v17, 128(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadReplicateRev {
+ size: 64,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(128).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ },
+ "E61020803805",
+ "vlbrrepg %v17, 128(%r2)",
+ ));
+
+ insns.push((
+ Inst::VecMov {
+ rd: writable_vr(8),
+ rn: vr(20),
+ },
+ "E78400000456",
+ "vlr %v8, %v20",
+ ));
+ insns.push((
+ Inst::VecCMov {
+ rd: writable_vr(8),
+ rm: vr(20),
+ cond: Cond::from_mask(1),
+ },
+ "A7E40005E78400000456",
+ "jno 10 ; vlr %v8, %v20",
+ ));
+ insns.push((
+ Inst::MovToVec128 {
+ rd: writable_vr(20),
+ rn: gpr(5),
+ rm: gpr(6),
+ },
+ "E74560000862",
+ "vlvgp %v20, %r5, %r6",
+ ));
+ insns.push((
+ Inst::VecLoadConst {
+ rd: writable_vr(24),
+ const_data: 0x0102030405060708090a0b0c0d0e0fu128,
+ },
+ "A715000A000102030405060708090A0B0C0D0E0FE78010000806",
+ "bras %r1, 20 ; data.u128 0x000102030405060708090a0b0c0d0e0f ; vl %v24, 0(%r1)",
+ ));
+ insns.push((
+ Inst::VecLoadConstReplicate {
+ size: 64,
+ rd: writable_vr(24),
+ const_data: 0x01020304050607u64,
+ },
+ "A71500060001020304050607E78010003805",
+ "bras %r1, 12 ; data.u64 0x0001020304050607 ; vlrepg %v24, 0(%r1)",
+ ));
+ insns.push((
+ Inst::VecLoadConstReplicate {
+ size: 32,
+ rd: writable_vr(24),
+ const_data: 0x010203u64,
+ },
+ "A715000400010203E78010002805",
+ "bras %r1, 8 ; data.u32 0x00010203 ; vlrepf %v24, 0(%r1)",
+ ));
+
+ insns.push((
+ Inst::VecImmByteMask {
+ rd: writable_vr(20),
+ mask: 0x1234,
+ },
+ "E74012340844",
+ "vgbm %v20, 4660",
+ ));
+ insns.push((
+ Inst::VecImmBitMask {
+ size: 8,
+ rd: writable_vr(20),
+ start_bit: 1,
+ end_bit: 7,
+ },
+ "E74001070846",
+ "vgmb %v20, 1, 7",
+ ));
+ insns.push((
+ Inst::VecImmBitMask {
+ size: 16,
+ rd: writable_vr(20),
+ start_bit: 1,
+ end_bit: 7,
+ },
+ "E74001071846",
+ "vgmh %v20, 1, 7",
+ ));
+ insns.push((
+ Inst::VecImmBitMask {
+ size: 32,
+ rd: writable_vr(20),
+ start_bit: 1,
+ end_bit: 7,
+ },
+ "E74001072846",
+ "vgmf %v20, 1, 7",
+ ));
+ insns.push((
+ Inst::VecImmBitMask {
+ size: 64,
+ rd: writable_vr(20),
+ start_bit: 1,
+ end_bit: 7,
+ },
+ "E74001073846",
+ "vgmg %v20, 1, 7",
+ ));
+ insns.push((
+ Inst::VecImmReplicate {
+ size: 8,
+ rd: writable_vr(20),
+ imm: 0x1234,
+ },
+ "E74012340845",
+ "vrepib %v20, 4660",
+ ));
+ insns.push((
+ Inst::VecImmReplicate {
+ size: 16,
+ rd: writable_vr(20),
+ imm: 0x1234,
+ },
+ "E74012341845",
+ "vrepih %v20, 4660",
+ ));
+ insns.push((
+ Inst::VecImmReplicate {
+ size: 32,
+ rd: writable_vr(20),
+ imm: 0x1234,
+ },
+ "E74012342845",
+ "vrepif %v20, 4660",
+ ));
+ insns.push((
+ Inst::VecImmReplicate {
+ size: 64,
+ rd: writable_vr(20),
+ imm: 0x1234,
+ },
+ "E74012343845",
+ "vrepig %v20, 4660",
+ ));
+
+ insns.push((
+ Inst::VecLoadLane {
+ size: 8,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 15,
+ },
+ "E7102000F800",
+ "vleb %v17, 0(%r2), 15",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 8,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7102FFF0800",
+ "vleb %v17, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 8,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 15,
+ },
+ "E7123000F800",
+ "vleb %v17, 0(%r2,%r3), 15",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 8,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7123FFF0800",
+ "vleb %v17, 4095(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 16,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 7,
+ },
+ "E71020007801",
+ "vleh %v17, 0(%r2), 7",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 16,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7102FFF0801",
+ "vleh %v17, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 16,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 7,
+ },
+ "E71230007801",
+ "vleh %v17, 0(%r2,%r3), 7",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 16,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7123FFF0801",
+ "vleh %v17, 4095(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 3,
+ },
+ "E71020003803",
+ "vlef %v17, 0(%r2), 3",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7102FFF0803",
+ "vlef %v17, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 3,
+ },
+ "E71230003803",
+ "vlef %v17, 0(%r2,%r3), 3",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7123FFF0803",
+ "vlef %v17, 4095(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 64,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 1,
+ },
+ "E71020001802",
+ "vleg %v17, 0(%r2), 1",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 64,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7102FFF0802",
+ "vleg %v17, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 64,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 1,
+ },
+ "E71230001802",
+ "vleg %v17, 0(%r2,%r3), 1",
+ ));
+ insns.push((
+ Inst::VecLoadLane {
+ size: 64,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7123FFF0802",
+ "vleg %v17, 4095(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "78102000",
+ "le %f1, 0(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "78102FFF",
+ "le %f1, 4095(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD20 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(-524288).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "ED1020008064",
+ "ley %f1, -524288(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD20 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(524287).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "ED102FFF7F64",
+ "ley %f1, 524287(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E71020000803",
+ "vlef %v17, 0(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7102FFF0803",
+ "vlef %v17, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "78123000",
+ "le %f1, 0(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "78123FFF",
+ "le %f1, 4095(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD20 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: SImm20::maybe_from_i64(-524288).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "ED1230008064",
+ "ley %f1, -524288(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD20 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: SImm20::maybe_from_i64(524287).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "ED123FFF7F64",
+ "ley %f1, 524287(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E71230000803",
+ "vlef %v17, 0(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7123FFF0803",
+ "vlef %v17, 4095(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 64,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "68102000",
+ "ld %f1, 0(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 64,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "68102FFF",
+ "ld %f1, 4095(%r2)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 64,
+ rd: writable_vr(1),
+ mem: MemArg::BXD20 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(-524288).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "ED1020008065",
+ "ldy %f1, -524288(%r2)",
+ ));
insns.push((
- Inst::FpuLoad64 {
+ Inst::VecLoadLaneUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -7708,12 +10752,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"ED102FFF7F65",
"ldy %f1, 524287(%r2)",
));
insns.push((
- Inst::FpuLoad64 {
+ Inst::VecLoadLaneUndef {
+ size: 64,
rd: writable_vr(17),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -7721,12 +10767,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E71020000802",
"vleg %v17, 0(%r2), 0",
));
insns.push((
- Inst::FpuLoad64 {
+ Inst::VecLoadLaneUndef {
+ size: 64,
rd: writable_vr(17),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -7734,12 +10782,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E7102FFF0802",
"vleg %v17, 4095(%r2), 0",
));
insns.push((
- Inst::FpuLoad64 {
+ Inst::VecLoadLaneUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -7747,77 +10797,209 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"68123000",
"ld %f1, 0(%r2,%r3)",
));
insns.push((
- Inst::FpuLoad64 {
+ Inst::VecLoadLaneUndef {
+ size: 64,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "68123FFF",
+ "ld %f1, 4095(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 64,
+ rd: writable_vr(1),
+ mem: MemArg::BXD20 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: SImm20::maybe_from_i64(-524288).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "ED1230008065",
+ "ldy %f1, -524288(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 64,
rd: writable_vr(1),
+ mem: MemArg::BXD20 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: SImm20::maybe_from_i64(524287).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "ED123FFF7F65",
+ "ldy %f1, 524287(%r2,%r3)",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 64,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E71230000802",
+ "vleg %v17, 0(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneUndef {
+ size: 64,
+ rd: writable_vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7123FFF0802",
+ "vleg %v17, 4095(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecStoreLane {
+ size: 8,
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 15,
+ },
+ "E7102000F808",
+ "vsteb %v17, 0(%r2), 15",
+ ));
+ insns.push((
+ Inst::VecStoreLane {
+ size: 8,
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E7102FFF0808",
+ "vsteb %v17, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecStoreLane {
+ size: 8,
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 15,
+ },
+ "E7123000F808",
+ "vsteb %v17, 0(%r2,%r3), 15",
+ ));
+ insns.push((
+ Inst::VecStoreLane {
+ size: 8,
+ rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(3),
index: gpr(2),
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
- "68123FFF",
- "ld %f1, 4095(%r2,%r3)",
+ "E7123FFF0808",
+ "vsteb %v17, 4095(%r2,%r3), 0",
));
insns.push((
- Inst::FpuLoad64 {
- rd: writable_vr(1),
- mem: MemArg::BXD20 {
- base: gpr(3),
- index: gpr(2),
- disp: SImm20::maybe_from_i64(-524288).unwrap(),
+ Inst::VecStoreLane {
+ size: 16,
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 7,
},
- "ED1230008065",
- "ldy %f1, -524288(%r2,%r3)",
+ "E71020007809",
+ "vsteh %v17, 0(%r2), 7",
));
insns.push((
- Inst::FpuLoad64 {
- rd: writable_vr(1),
- mem: MemArg::BXD20 {
- base: gpr(3),
- index: gpr(2),
- disp: SImm20::maybe_from_i64(524287).unwrap(),
+ Inst::VecStoreLane {
+ size: 16,
+ rd: vr(17),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
- "ED123FFF7F65",
- "ldy %f1, 524287(%r2,%r3)",
+ "E7102FFF0809",
+ "vsteh %v17, 4095(%r2), 0",
));
insns.push((
- Inst::FpuLoad64 {
- rd: writable_vr(17),
+ Inst::VecStoreLane {
+ size: 16,
+ rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(3),
index: gpr(2),
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 7,
},
- "E71230000802",
- "vleg %v17, 0(%r2,%r3), 0",
+ "E71230007809",
+ "vsteh %v17, 0(%r2,%r3), 7",
));
insns.push((
- Inst::FpuLoad64 {
- rd: writable_vr(17),
+ Inst::VecStoreLane {
+ size: 16,
+ rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(3),
index: gpr(2),
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
- "E7123FFF0802",
- "vleg %v17, 4095(%r2,%r3), 0",
+ "E7123FFF0809",
+ "vsteh %v17, 4095(%r2,%r3), 0",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -7825,12 +11007,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"70102000",
"ste %f1, 0(%r2)",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -7838,12 +11022,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"70102FFF",
"ste %f1, 4095(%r2)",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -7851,12 +11037,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"ED1020008066",
"stey %f1, -524288(%r2)",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -7864,12 +11052,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"ED102FFF7F66",
"stey %f1, 524287(%r2)",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -7877,12 +11067,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E7102000080B",
"vstef %v17, 0(%r2), 0",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -7890,12 +11082,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E7102FFF080B",
"vstef %v17, 4095(%r2), 0",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -7903,12 +11097,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"70123000",
"ste %f1, 0(%r2,%r3)",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -7916,12 +11112,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"70123FFF",
"ste %f1, 4095(%r2,%r3)",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -7929,12 +11127,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"ED1230008066",
"stey %f1, -524288(%r2,%r3)",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -7942,12 +11142,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"ED123FFF7F66",
"stey %f1, 524287(%r2,%r3)",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -7955,12 +11157,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E7123000080B",
"vstef %v17, 0(%r2,%r3), 0",
));
insns.push((
- Inst::FpuStore32 {
+ Inst::VecStoreLane {
+ size: 32,
rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -7968,12 +11172,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E7123FFF080B",
"vstef %v17, 4095(%r2,%r3), 0",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -7981,12 +11187,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"60102000",
"std %f1, 0(%r2)",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -7994,12 +11202,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"60102FFF",
"std %f1, 4095(%r2)",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8007,12 +11217,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"ED1020008067",
"stdy %f1, -524288(%r2)",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8020,12 +11232,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"ED102FFF7F67",
"stdy %f1, 524287(%r2)",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8033,12 +11247,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E7102000080A",
"vsteg %v17, 0(%r2), 0",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8046,12 +11262,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E7102FFF080A",
"vsteg %v17, 4095(%r2), 0",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8059,12 +11277,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"60123000",
"std %f1, 0(%r2,%r3)",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8072,12 +11292,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"60123FFF",
"std %f1, 4095(%r2,%r3)",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8085,12 +11307,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"ED1230008067",
"stdy %f1, -524288(%r2,%r3)",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8098,12 +11322,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"ED123FFF7F67",
"stdy %f1, 524287(%r2,%r3)",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8111,12 +11337,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E7123000080A",
"vsteg %v17, 0(%r2,%r3), 0",
));
insns.push((
- Inst::FpuStore64 {
+ Inst::VecStoreLane {
+ size: 64,
rd: vr(17),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8124,13 +11352,194 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E7123FFF080A",
"vsteg %v17, 4095(%r2,%r3), 0",
));
-
insns.push((
- Inst::FpuLoadRev32 {
+ Inst::VecLoadLaneRev {
+ size: 16,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E61020000001",
+ "vlebrh %v1, 0(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 16,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E6102FFF0001",
+ "vlebrh %v1, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 16,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E61230000001",
+ "vlebrh %v1, 0(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 16,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E6123FFF0001",
+ "vlebrh %v1, 4095(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E61020000003",
+ "vlebrf %v1, 0(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E6102FFF0003",
+ "vlebrf %v1, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E61230000003",
+ "vlebrf %v1, 0(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 32,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E6123FFF0003",
+ "vlebrf %v1, 4095(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 64,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E61020000002",
+ "vlebrg %v1, 0(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 64,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E6102FFF0002",
+ "vlebrg %v1, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 64,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E61230000002",
+ "vlebrg %v1, 0(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRev {
+ size: 64,
+ rd: writable_vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E6123FFF0002",
+ "vlebrg %v1, 4095(%r2,%r3), 0",
+ ));
+ insns.push((
+ Inst::VecLoadLaneRevUndef {
+ size: 32,
rd: writable_vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8138,12 +11547,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E61020000003",
- "vlebrf %f1, 0(%r2), 0",
+ "vlebrf %v1, 0(%r2), 0",
));
insns.push((
- Inst::FpuLoadRev32 {
+ Inst::VecLoadLaneRevUndef {
+ size: 32,
rd: writable_vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8151,12 +11562,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6102FFF0003",
- "vlebrf %f1, 4095(%r2), 0",
+ "vlebrf %v1, 4095(%r2), 0",
));
insns.push((
- Inst::FpuLoadRev32 {
+ Inst::VecLoadLaneRevUndef {
+ size: 32,
rd: writable_vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8164,12 +11577,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E31020008071E61010000003",
- "lay %r1, -524288(%r2) ; vlebrf %f1, 0(%r1), 0",
+ "lay %r1, -524288(%r2) ; vlebrf %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuLoadRev32 {
+ Inst::VecLoadLaneRevUndef {
+ size: 32,
rd: writable_vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8177,12 +11592,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E3102FFF7F71E61010000003",
- "lay %r1, 524287(%r2) ; vlebrf %f1, 0(%r1), 0",
+ "lay %r1, 524287(%r2) ; vlebrf %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuLoadRev32 {
+ Inst::VecLoadLaneRevUndef {
+ size: 32,
rd: writable_vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8190,12 +11607,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E61230000003",
- "vlebrf %f1, 0(%r2,%r3), 0",
+ "vlebrf %v1, 0(%r2,%r3), 0",
));
insns.push((
- Inst::FpuLoadRev32 {
+ Inst::VecLoadLaneRevUndef {
+ size: 32,
rd: writable_vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8203,12 +11622,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6123FFF0003",
- "vlebrf %f1, 4095(%r2,%r3), 0",
+ "vlebrf %v1, 4095(%r2,%r3), 0",
));
insns.push((
- Inst::FpuLoadRev32 {
+ Inst::VecLoadLaneRevUndef {
+ size: 32,
rd: writable_vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8216,12 +11637,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E31230008071E61010000003",
- "lay %r1, -524288(%r2,%r3) ; vlebrf %f1, 0(%r1), 0",
+ "lay %r1, -524288(%r2,%r3) ; vlebrf %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuLoadRev32 {
+ Inst::VecLoadLaneRevUndef {
+ size: 32,
rd: writable_vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8229,12 +11652,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E3123FFF7F71E61010000003",
- "lay %r1, 524287(%r2,%r3) ; vlebrf %f1, 0(%r1), 0",
+ "lay %r1, 524287(%r2,%r3) ; vlebrf %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuLoadRev64 {
+ Inst::VecLoadLaneRevUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8242,12 +11667,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E61020000002",
- "vlebrg %f1, 0(%r2), 0",
+ "vlebrg %v1, 0(%r2), 0",
));
insns.push((
- Inst::FpuLoadRev64 {
+ Inst::VecLoadLaneRevUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8255,12 +11682,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6102FFF0002",
- "vlebrg %f1, 4095(%r2), 0",
+ "vlebrg %v1, 4095(%r2), 0",
));
insns.push((
- Inst::FpuLoadRev64 {
+ Inst::VecLoadLaneRevUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8268,12 +11697,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E31020008071E61010000002",
- "lay %r1, -524288(%r2) ; vlebrg %f1, 0(%r1), 0",
+ "lay %r1, -524288(%r2) ; vlebrg %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuLoadRev64 {
+ Inst::VecLoadLaneRevUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8281,12 +11712,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E3102FFF7F71E61010000002",
- "lay %r1, 524287(%r2) ; vlebrg %f1, 0(%r1), 0",
+ "lay %r1, 524287(%r2) ; vlebrg %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuLoadRev64 {
+ Inst::VecLoadLaneRevUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8294,12 +11727,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E61230000002",
- "vlebrg %f1, 0(%r2,%r3), 0",
+ "vlebrg %v1, 0(%r2,%r3), 0",
));
insns.push((
- Inst::FpuLoadRev64 {
+ Inst::VecLoadLaneRevUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8307,12 +11742,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6123FFF0002",
- "vlebrg %f1, 4095(%r2,%r3), 0",
+ "vlebrg %v1, 4095(%r2,%r3), 0",
));
insns.push((
- Inst::FpuLoadRev64 {
+ Inst::VecLoadLaneRevUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8320,12 +11757,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E31230008071E61010000002",
- "lay %r1, -524288(%r2,%r3) ; vlebrg %f1, 0(%r1), 0",
+ "lay %r1, -524288(%r2,%r3) ; vlebrg %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuLoadRev64 {
+ Inst::VecLoadLaneRevUndef {
+ size: 64,
rd: writable_vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8333,12 +11772,74 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E3123FFF7F71E61010000002",
- "lay %r1, 524287(%r2,%r3) ; vlebrg %f1, 0(%r1), 0",
+ "lay %r1, 524287(%r2,%r3) ; vlebrg %v1, 0(%r1), 0",
+ ));
+ insns.push((
+ Inst::VecStoreLaneRev {
+ size: 16,
+ rd: vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 7,
+ },
+ "E61020007009",
+ "vstebrh %v1, 0(%r2), 7",
+ ));
+ insns.push((
+ Inst::VecStoreLaneRev {
+ size: 16,
+ rd: vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(2),
+ index: zero_reg(),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E6102FFF0009",
+ "vstebrh %v1, 4095(%r2), 0",
+ ));
+ insns.push((
+ Inst::VecStoreLaneRev {
+ size: 16,
+ rd: vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::zero(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 7,
+ },
+ "E61230007009",
+ "vstebrh %v1, 0(%r2,%r3), 7",
+ ));
+ insns.push((
+ Inst::VecStoreLaneRev {
+ size: 16,
+ rd: vr(1),
+ mem: MemArg::BXD12 {
+ base: gpr(3),
+ index: gpr(2),
+ disp: UImm12::maybe_from_u64(4095).unwrap(),
+ flags: MemFlags::trusted(),
+ },
+ lane_imm: 0,
+ },
+ "E6123FFF0009",
+ "vstebrh %v1, 4095(%r2,%r3), 0",
));
insns.push((
- Inst::FpuStoreRev32 {
+ Inst::VecStoreLaneRev {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8346,12 +11847,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6102000000B",
- "vstebrf %f1, 0(%r2), 0",
+ "vstebrf %v1, 0(%r2), 0",
));
insns.push((
- Inst::FpuStoreRev32 {
+ Inst::VecStoreLaneRev {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8359,12 +11862,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6102FFF000B",
- "vstebrf %f1, 4095(%r2), 0",
+ "vstebrf %v1, 4095(%r2), 0",
));
insns.push((
- Inst::FpuStoreRev32 {
+ Inst::VecStoreLaneRev {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8372,12 +11877,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E31020008071E6101000000B",
- "lay %r1, -524288(%r2) ; vstebrf %f1, 0(%r1), 0",
+ "lay %r1, -524288(%r2) ; vstebrf %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuStoreRev32 {
+ Inst::VecStoreLaneRev {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8385,12 +11892,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E3102FFF7F71E6101000000B",
- "lay %r1, 524287(%r2) ; vstebrf %f1, 0(%r1), 0",
+ "lay %r1, 524287(%r2) ; vstebrf %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuStoreRev32 {
+ Inst::VecStoreLaneRev {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8398,12 +11907,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6123000000B",
- "vstebrf %f1, 0(%r2,%r3), 0",
+ "vstebrf %v1, 0(%r2,%r3), 0",
));
insns.push((
- Inst::FpuStoreRev32 {
+ Inst::VecStoreLaneRev {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8411,12 +11922,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6123FFF000B",
- "vstebrf %f1, 4095(%r2,%r3), 0",
+ "vstebrf %v1, 4095(%r2,%r3), 0",
));
insns.push((
- Inst::FpuStoreRev32 {
+ Inst::VecStoreLaneRev {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8424,12 +11937,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E31230008071E6101000000B",
- "lay %r1, -524288(%r2,%r3) ; vstebrf %f1, 0(%r1), 0",
+ "lay %r1, -524288(%r2,%r3) ; vstebrf %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuStoreRev32 {
+ Inst::VecStoreLaneRev {
+ size: 32,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8437,12 +11952,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E3123FFF7F71E6101000000B",
- "lay %r1, 524287(%r2,%r3) ; vstebrf %f1, 0(%r1), 0",
+ "lay %r1, 524287(%r2,%r3) ; vstebrf %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuStoreRev64 {
+ Inst::VecStoreLaneRev {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8450,12 +11967,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6102000000A",
- "vstebrg %f1, 0(%r2), 0",
+ "vstebrg %v1, 0(%r2), 0",
));
insns.push((
- Inst::FpuStoreRev64 {
+ Inst::VecStoreLaneRev {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(2),
@@ -8463,12 +11982,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6102FFF000A",
- "vstebrg %f1, 4095(%r2), 0",
+ "vstebrg %v1, 4095(%r2), 0",
));
insns.push((
- Inst::FpuStoreRev64 {
+ Inst::VecStoreLaneRev {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8476,12 +11997,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E31020008071E6101000000A",
- "lay %r1, -524288(%r2) ; vstebrg %f1, 0(%r1), 0",
+ "lay %r1, -524288(%r2) ; vstebrg %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuStoreRev64 {
+ Inst::VecStoreLaneRev {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(2),
@@ -8489,12 +12012,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E3102FFF7F71E6101000000A",
- "lay %r1, 524287(%r2) ; vstebrg %f1, 0(%r1), 0",
+ "lay %r1, 524287(%r2) ; vstebrg %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuStoreRev64 {
+ Inst::VecStoreLaneRev {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8502,12 +12027,14 @@ fn test_s390x_binemit() {
disp: UImm12::zero(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6123000000A",
- "vstebrg %f1, 0(%r2,%r3), 0",
+ "vstebrg %v1, 0(%r2,%r3), 0",
));
insns.push((
- Inst::FpuStoreRev64 {
+ Inst::VecStoreLaneRev {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD12 {
base: gpr(3),
@@ -8515,12 +12042,14 @@ fn test_s390x_binemit() {
disp: UImm12::maybe_from_u64(4095).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E6123FFF000A",
- "vstebrg %f1, 4095(%r2,%r3), 0",
+ "vstebrg %v1, 4095(%r2,%r3), 0",
));
insns.push((
- Inst::FpuStoreRev64 {
+ Inst::VecStoreLaneRev {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8528,12 +12057,14 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(-524288).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E31230008071E6101000000A",
- "lay %r1, -524288(%r2,%r3) ; vstebrg %f1, 0(%r1), 0",
+ "lay %r1, -524288(%r2,%r3) ; vstebrg %v1, 0(%r1), 0",
));
insns.push((
- Inst::FpuStoreRev64 {
+ Inst::VecStoreLaneRev {
+ size: 64,
rd: vr(1),
mem: MemArg::BXD20 {
base: gpr(3),
@@ -8541,304 +12072,476 @@ fn test_s390x_binemit() {
disp: SImm20::maybe_from_i64(524287).unwrap(),
flags: MemFlags::trusted(),
},
+ lane_imm: 0,
},
"E3123FFF7F71E6101000000A",
- "lay %r1, 524287(%r2,%r3) ; vstebrg %f1, 0(%r1), 0",
+ "lay %r1, 524287(%r2,%r3) ; vstebrg %v1, 0(%r1), 0",
));
insns.push((
- Inst::LoadFpuConst32 {
+ Inst::VecInsertLane {
+ size: 8,
+ rd: writable_vr(8),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
+ },
+ "E78400000022",
+ "vlvgb %v8, %r4, 0",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 8,
+ rd: writable_vr(8),
+ rn: gpr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
+ },
+ "E78400FF0022",
+ "vlvgb %v8, %r4, 255",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 8,
+ rd: writable_vr(24),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: gpr(3),
+ },
+ "E78430000822",
+ "vlvgb %v24, %r4, 0(%r3)",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 16,
+ rd: writable_vr(8),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
+ },
+ "E78400001022",
+ "vlvgh %v8, %r4, 0",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 16,
+ rd: writable_vr(8),
+ rn: gpr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
+ },
+ "E78400FF1022",
+ "vlvgh %v8, %r4, 255",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 16,
+ rd: writable_vr(24),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: gpr(3),
+ },
+ "E78430001822",
+ "vlvgh %v24, %r4, 0(%r3)",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 32,
+ rd: writable_vr(8),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
+ },
+ "E78400002022",
+ "vlvgf %v8, %r4, 0",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 32,
+ rd: writable_vr(8),
+ rn: gpr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
+ },
+ "E78400FF2022",
+ "vlvgf %v8, %r4, 255",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 32,
+ rd: writable_vr(24),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: gpr(3),
+ },
+ "E78430002822",
+ "vlvgf %v24, %r4, 0(%r3)",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 64,
+ rd: writable_vr(8),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
+ },
+ "E78400003022",
+ "vlvgg %v8, %r4, 0",
+ ));
+ insns.push((
+ Inst::VecInsertLane {
+ size: 64,
rd: writable_vr(8),
- const_data: 1.0_f32.to_bits(),
+ rn: gpr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
},
- "A71500043F80000078801000",
- "bras %r1, 8 ; data.f32 1 ; le %f8, 0(%r1)",
+ "E78400FF3022",
+ "vlvgg %v8, %r4, 255",
));
insns.push((
- Inst::LoadFpuConst32 {
+ Inst::VecInsertLane {
+ size: 64,
rd: writable_vr(24),
- const_data: 1.0_f32.to_bits(),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: gpr(3),
},
- "A71500043F800000E78010000803",
- "bras %r1, 8 ; data.f32 1 ; vlef %v24, 0(%r1), 0",
+ "E78430003822",
+ "vlvgg %v24, %r4, 0(%r3)",
));
insns.push((
- Inst::LoadFpuConst64 {
+ Inst::VecInsertLaneUndef {
+ size: 8,
rd: writable_vr(8),
- const_data: 1.0_f64.to_bits(),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
},
- "A71500063FF000000000000068801000",
- "bras %r1, 12 ; data.f64 1 ; ld %f8, 0(%r1)",
+ "E78400000022",
+ "vlvgb %v8, %r4, 0",
));
insns.push((
- Inst::LoadFpuConst64 {
+ Inst::VecInsertLaneUndef {
+ size: 8,
+ rd: writable_vr(8),
+ rn: gpr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
+ },
+ "E78400FF0022",
+ "vlvgb %v8, %r4, 255",
+ ));
+ insns.push((
+ Inst::VecInsertLaneUndef {
+ size: 8,
rd: writable_vr(24),
- const_data: 1.0_f64.to_bits(),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: gpr(3),
},
- "A71500063FF0000000000000E78010000802",
- "bras %r1, 12 ; data.f64 1 ; vleg %v24, 0(%r1), 0",
+ "E78430000822",
+ "vlvgb %v24, %r4, 0(%r3)",
));
-
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Cvt64To32,
- mode: FpuRoundMode::Current,
+ Inst::VecInsertLaneUndef {
+ size: 16,
rd: writable_vr(8),
- rn: vr(12),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
},
- "B344008C",
- "ledbra %f8, %f12, 0",
+ "E78400001022",
+ "vlvgh %v8, %r4, 0",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Cvt64To32,
- mode: FpuRoundMode::ToNearest,
+ Inst::VecInsertLaneUndef {
+ size: 16,
+ rd: writable_vr(8),
+ rn: gpr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
+ },
+ "E78400FF1022",
+ "vlvgh %v8, %r4, 255",
+ ));
+ insns.push((
+ Inst::VecInsertLaneUndef {
+ size: 16,
rd: writable_vr(24),
- rn: vr(12),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: gpr(3),
},
- "E78C001838C5",
- "wledb %v24, %f12, 0, 1",
+ "E78430001822",
+ "vlvgh %v24, %r4, 0(%r3)",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round32,
- mode: FpuRoundMode::ToNegInfinity,
+ Inst::VecInsertLaneUndef {
+ size: 32,
rd: writable_vr(8),
- rn: vr(12),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
},
- "B357708C",
- "fiebr %f8, %f12, 7",
+ "E78400002022",
+ "vlvgf %v8, %r4, 0",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round64,
- mode: FpuRoundMode::ToNegInfinity,
+ Inst::VecInsertLaneUndef {
+ size: 32,
rd: writable_vr(8),
- rn: vr(12),
+ rn: gpr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
},
- "B35F708C",
- "fidbr %f8, %f12, 7",
+ "E78400FF2022",
+ "vlvgf %v8, %r4, 255",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round32,
- mode: FpuRoundMode::ToPosInfinity,
- rd: writable_vr(8),
- rn: vr(12),
+ Inst::VecInsertLaneUndef {
+ size: 32,
+ rd: writable_vr(24),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: gpr(3),
},
- "B357608C",
- "fiebr %f8, %f12, 6",
+ "E78430002822",
+ "vlvgf %v24, %r4, 0(%r3)",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round64,
- mode: FpuRoundMode::ToPosInfinity,
+ Inst::VecInsertLaneUndef {
+ size: 64,
rd: writable_vr(8),
- rn: vr(12),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
},
- "B35F608C",
- "fidbr %f8, %f12, 6",
+ "B3C10084",
+ "ldgr %f8, %r4",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round32,
- mode: FpuRoundMode::ToZero,
+ Inst::VecInsertLaneUndef {
+ size: 64,
rd: writable_vr(8),
- rn: vr(12),
+ rn: gpr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
},
- "B357508C",
- "fiebr %f8, %f12, 5",
+ "E78400FF3022",
+ "vlvgg %v8, %r4, 255",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round64,
- mode: FpuRoundMode::ToZero,
+ Inst::VecInsertLaneUndef {
+ size: 64,
rd: writable_vr(8),
- rn: vr(12),
+ rn: gpr(4),
+ lane_imm: 0,
+ lane_reg: gpr(3),
},
- "B35F508C",
- "fidbr %f8, %f12, 5",
+ "E78430003022",
+ "vlvgg %v8, %r4, 0(%r3)",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round32,
- mode: FpuRoundMode::ToNearestTiesToEven,
- rd: writable_vr(8),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 8,
+ rd: writable_gpr(8),
+ rn: vr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
},
- "B357408C",
- "fiebr %f8, %f12, 4",
+ "E78400FF0021",
+ "vlgvb %r8, %v4, 255",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round64,
- mode: FpuRoundMode::ToNearestTiesToEven,
- rd: writable_vr(8),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 8,
+ rd: writable_gpr(8),
+ rn: vr(20),
+ lane_imm: 0,
+ lane_reg: gpr(3),
},
- "B35F408C",
- "fidbr %f8, %f12, 4",
+ "E78430000421",
+ "vlgvb %r8, %v20, 0(%r3)",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round32,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 16,
+ rd: writable_gpr(8),
+ rn: vr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
},
- "E78C001828C7",
- "wfisb %v24, %f12, 0, 1",
+ "E78400001021",
+ "vlgvh %r8, %v4, 0",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::Round64,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 16,
+ rd: writable_gpr(8),
+ rn: vr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
},
- "E78C001838C7",
- "wfidb %v24, %f12, 0, 1",
+ "E78400FF1021",
+ "vlgvh %r8, %v4, 255",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::ToSInt32,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 16,
+ rd: writable_gpr(8),
+ rn: vr(20),
+ lane_imm: 0,
+ lane_reg: gpr(3),
},
- "E78C001828C2",
- "wcfeb %v24, %f12, 0, 1",
+ "E78430001421",
+ "vlgvh %r8, %v20, 0(%r3)",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::ToSInt64,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 32,
+ rd: writable_gpr(8),
+ rn: vr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
},
- "E78C001838C2",
- "wcgdb %v24, %f12, 0, 1",
+ "E78400002021",
+ "vlgvf %r8, %v4, 0",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::ToUInt32,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 32,
+ rd: writable_gpr(8),
+ rn: vr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
},
- "E78C001828C0",
- "wclfeb %v24, %f12, 0, 1",
+ "E78400FF2021",
+ "vlgvf %r8, %v4, 255",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::ToUInt64,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 32,
+ rd: writable_gpr(8),
+ rn: vr(20),
+ lane_imm: 0,
+ lane_reg: gpr(3),
},
- "E78C001838C0",
- "wclgdb %v24, %f12, 0, 1",
+ "E78430002421",
+ "vlgvf %r8, %v20, 0(%r3)",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::FromSInt32,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 64,
+ rd: writable_gpr(8),
+ rn: vr(4),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
},
- "E78C001828C3",
- "wcefb %v24, %f12, 0, 1",
+ "B3CD0084",
+ "lgdr %r8, %f4",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::FromSInt64,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 64,
+ rd: writable_gpr(8),
+ rn: vr(4),
+ lane_imm: 255,
+ lane_reg: zero_reg(),
},
- "E78C001838C3",
- "wcdgb %v24, %f12, 0, 1",
+ "E78400FF3021",
+ "vlgvg %r8, %v4, 255",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::FromUInt32,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecExtractLane {
+ size: 64,
+ rd: writable_gpr(8),
+ rn: vr(4),
+ lane_imm: 0,
+ lane_reg: gpr(3),
},
- "E78C001828C1",
- "wcelfb %v24, %f12, 0, 1",
+ "E78430003021",
+ "vlgvg %r8, %v4, 0(%r3)",
));
insns.push((
- Inst::FpuRound {
- op: FpuRoundOp::FromUInt64,
- mode: FpuRoundMode::ToNearest,
- rd: writable_vr(24),
- rn: vr(12),
+ Inst::VecInsertLaneImm {
+ size: 8,
+ rd: writable_vr(20),
+ imm: 0x1234,
+ lane_imm: 15,
},
- "E78C001838C1",
- "wcdlgb %v24, %f12, 0, 1",
+ "E7401234F840",
+ "vleib %v20, 4660, 15",
));
-
insns.push((
- Inst::VecSelect {
- rd: writable_vr(4),
- rn: vr(6),
- rm: vr(8),
- ra: vr(10),
+ Inst::VecInsertLaneImm {
+ size: 16,
+ rd: writable_vr(20),
+ imm: 0x1234,
+ lane_imm: 7,
},
- "E7468000A08D",
- "vsel %v4, %v6, %v8, %v10",
+ "E74012347841",
+ "vleih %v20, 4660, 7",
));
insns.push((
- Inst::VecSelect {
+ Inst::VecInsertLaneImm {
+ size: 32,
rd: writable_vr(20),
- rn: vr(6),
- rm: vr(8),
- ra: vr(10),
+ imm: 0x1234,
+ lane_imm: 3,
},
- "E7468000A88D",
- "vsel %v20, %v6, %v8, %v10",
+ "E74012343843",
+ "vleif %v20, 4660, 3",
));
insns.push((
- Inst::VecSelect {
- rd: writable_vr(4),
- rn: vr(22),
- rm: vr(8),
- ra: vr(10),
+ Inst::VecInsertLaneImm {
+ size: 64,
+ rd: writable_vr(20),
+ imm: 0x1234,
+ lane_imm: 1,
},
- "E7468000A48D",
- "vsel %v4, %v22, %v8, %v10",
+ "E74012341842",
+ "vleig %v20, 4660, 1",
));
insns.push((
- Inst::VecSelect {
- rd: writable_vr(4),
- rn: vr(6),
- rm: vr(24),
- ra: vr(10),
+ Inst::VecReplicateLane {
+ size: 8,
+ rd: writable_vr(20),
+ rn: vr(8),
+ lane_imm: 15,
},
- "E7468000A28D",
- "vsel %v4, %v6, %v24, %v10",
+ "E748000F084D",
+ "vrepb %v20, %v8, 15",
));
insns.push((
- Inst::VecSelect {
- rd: writable_vr(4),
- rn: vr(6),
- rm: vr(8),
- ra: vr(26),
+ Inst::VecReplicateLane {
+ size: 16,
+ rd: writable_vr(20),
+ rn: vr(8),
+ lane_imm: 7,
},
- "E7468000A18D",
- "vsel %v4, %v6, %v8, %v26",
+ "E7480007184D",
+ "vreph %v20, %v8, 7",
));
insns.push((
- Inst::VecSelect {
+ Inst::VecReplicateLane {
+ size: 32,
rd: writable_vr(20),
- rn: vr(22),
- rm: vr(24),
- ra: vr(26),
+ rn: vr(8),
+ lane_imm: 3,
},
- "E7468000AF8D",
- "vsel %v20, %v22, %v24, %v26",
+ "E7480003284D",
+ "vrepf %v20, %v8, 3",
+ ));
+ insns.push((
+ Inst::VecReplicateLane {
+ size: 64,
+ rd: writable_vr(20),
+ rn: vr(8),
+ lane_imm: 1,
+ },
+ "E7480001384D",
+ "vrepg %v20, %v8, 1",
));
let flags = settings::Flags::new(settings::builder());
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index 676e0d4794f4..ad5af092bc6b 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -2,6 +2,7 @@
use crate::binemit::{Addend, CodeOffset, Reloc};
use crate::ir::{types, ExternalName, Opcode, Type};
+use crate::isa::CallConv;
use crate::machinst::*;
use crate::{settings, CodegenError, CodegenResult};
use alloc::boxed::Box;
@@ -28,7 +29,7 @@ mod emit_tests;
pub use crate::isa::s390x::lower::isle::generated_code::{
ALUOp, CmpOp, FPUOp1, FPUOp2, FPUOp3, FpuRoundMode, FpuRoundOp, MInst as Inst, RxSBGOp,
- ShiftOp, UnaryOp,
+ ShiftOp, UnaryOp, VecBinaryOp, VecFloatCmpOp, VecIntCmpOp, VecShiftOp, VecUnaryOp,
};
/// Additional information for (direct) Call instructions, left out of line to lower the size of
@@ -40,6 +41,8 @@ pub struct CallInfo {
pub defs: SmallVec<[Writable<Reg>; 8]>,
pub clobbers: PRegSet,
pub opcode: Opcode,
+ pub caller_callconv: CallConv,
+ pub callee_callconv: CallConv,
}
/// Additional information for CallInd instructions, left out of line to lower the size of the Inst
@@ -51,6 +54,8 @@ pub struct CallIndInfo {
pub defs: SmallVec<[Writable<Reg>; 8]>,
pub clobbers: PRegSet,
pub opcode: Opcode,
+ pub caller_callconv: CallConv,
+ pub callee_callconv: CallConv,
}
#[test]
@@ -156,22 +161,42 @@ impl Inst {
| Inst::FpuMove64 { .. }
| Inst::FpuCMov32 { .. }
| Inst::FpuCMov64 { .. }
- | Inst::MovToFpr32 { .. }
- | Inst::MovToFpr64 { .. }
- | Inst::MovFromFpr32 { .. }
- | Inst::MovFromFpr64 { .. }
| Inst::FpuRR { .. }
| Inst::FpuRRR { .. }
| Inst::FpuRRRR { .. }
| Inst::FpuCmp32 { .. }
| Inst::FpuCmp64 { .. }
- | Inst::FpuLoad32 { .. }
- | Inst::FpuStore32 { .. }
- | Inst::FpuLoad64 { .. }
- | Inst::FpuStore64 { .. }
| Inst::LoadFpuConst32 { .. }
| Inst::LoadFpuConst64 { .. }
+ | Inst::VecRRR { .. }
+ | Inst::VecRR { .. }
+ | Inst::VecShiftRR { .. }
| Inst::VecSelect { .. }
+ | Inst::VecPermute { .. }
+ | Inst::VecPermuteDWImm { .. }
+ | Inst::VecIntCmp { .. }
+ | Inst::VecIntCmpS { .. }
+ | Inst::VecFloatCmp { .. }
+ | Inst::VecFloatCmpS { .. }
+ | Inst::VecLoad { .. }
+ | Inst::VecStore { .. }
+ | Inst::VecLoadReplicate { .. }
+ | Inst::VecMov { .. }
+ | Inst::VecCMov { .. }
+ | Inst::MovToVec128 { .. }
+ | Inst::VecLoadConst { .. }
+ | Inst::VecLoadConstReplicate { .. }
+ | Inst::VecImmByteMask { .. }
+ | Inst::VecImmBitMask { .. }
+ | Inst::VecImmReplicate { .. }
+ | Inst::VecLoadLane { .. }
+ | Inst::VecLoadLaneUndef { .. }
+ | Inst::VecStoreLane { .. }
+ | Inst::VecInsertLane { .. }
+ | Inst::VecInsertLaneUndef { .. }
+ | Inst::VecExtractLane { .. }
+ | Inst::VecInsertLaneImm { .. }
+ | Inst::VecReplicateLane { .. }
| Inst::Call { .. }
| Inst::CallInd { .. }
| Inst::Ret { .. }
@@ -207,19 +232,33 @@ impl Inst {
Inst::FpuRound { op, .. } => match op {
FpuRoundOp::ToSInt32 | FpuRoundOp::FromSInt32 => InstructionSet::MIE2,
FpuRoundOp::ToUInt32 | FpuRoundOp::FromUInt32 => InstructionSet::MIE2,
+ FpuRoundOp::ToSInt32x4 | FpuRoundOp::FromSInt32x4 => InstructionSet::MIE2,
+ FpuRoundOp::ToUInt32x4 | FpuRoundOp::FromUInt32x4 => InstructionSet::MIE2,
_ => InstructionSet::Base,
},
// These are all part of VXRS_EXT2
- Inst::FpuLoadRev32 { .. }
- | Inst::FpuStoreRev32 { .. }
- | Inst::FpuLoadRev64 { .. }
- | Inst::FpuStoreRev64 { .. } => InstructionSet::VXRS_EXT2,
+ Inst::VecLoadRev { .. }
+ | Inst::VecStoreRev { .. }
+ | Inst::VecLoadReplicateRev { .. }
+ | Inst::VecLoadLaneRev { .. }
+ | Inst::VecLoadLaneRevUndef { .. }
+ | Inst::VecStoreLaneRev { .. } => InstructionSet::VXRS_EXT2,
Inst::DummyUse { .. } => InstructionSet::Base,
}
}
+ /// Create a 128-bit move instruction.
+ pub fn mov128(to_reg: Writable<Reg>, from_reg: Reg) -> Inst {
+ assert!(to_reg.to_reg().class() == RegClass::Float);
+ assert!(from_reg.class() == RegClass::Float);
+ Inst::VecMov {
+ rd: to_reg,
+ rn: from_reg,
+ }
+ }
+
/// Create a 64-bit move instruction.
pub fn mov64(to_reg: Writable<Reg>, from_reg: Reg) -> Inst {
assert!(to_reg.to_reg().class() == from_reg.class());
@@ -323,6 +362,17 @@ impl Inst {
}
}
+ /// Create an instruction that loads a 128-bit floating-point constant.
+ pub fn load_vec_constant(rd: Writable<Reg>, value: u128) -> Inst {
+ // FIXME: This doesn't special-case constants that can be loaded
+ // without a constant pool, like the ISLE lowering does. Ideally,
+ // we should not have to duplicate the logic here.
+ Inst::VecLoadConst {
+ rd,
+ const_data: value,
+ }
+ }
+
/// Generic constructor for a load (zero-extending where appropriate).
pub fn gen_load(into_reg: Writable<Reg>, mem: MemArg, ty: Type) -> Inst {
match ty {
@@ -330,8 +380,19 @@ impl Inst {
types::B16 | types::I16 => Inst::Load64ZExt16 { rd: into_reg, mem },
types::B32 | types::I32 => Inst::Load64ZExt32 { rd: into_reg, mem },
types::B64 | types::I64 | types::R64 => Inst::Load64 { rd: into_reg, mem },
- types::F32 => Inst::FpuLoad32 { rd: into_reg, mem },
- types::F64 => Inst::FpuLoad64 { rd: into_reg, mem },
+ types::F32 => Inst::VecLoadLaneUndef {
+ size: 32,
+ rd: into_reg,
+ mem,
+ lane_imm: 0,
+ },
+ types::F64 => Inst::VecLoadLaneUndef {
+ size: 64,
+ rd: into_reg,
+ mem,
+ lane_imm: 0,
+ },
+ _ if ty.is_vector() && ty.bits() == 128 => Inst::VecLoad { rd: into_reg, mem },
_ => unimplemented!("gen_load({})", ty),
}
}
@@ -343,8 +404,19 @@ impl Inst {
types::B16 | types::I16 => Inst::Store16 { rd: from_reg, mem },
types::B32 | types::I32 => Inst::Store32 { rd: from_reg, mem },
types::B64 | types::I64 | types::R64 => Inst::Store64 { rd: from_reg, mem },
- types::F32 => Inst::FpuStore32 { rd: from_reg, mem },
- types::F64 => Inst::FpuStore64 { rd: from_reg, mem },
+ types::F32 => Inst::VecStoreLane {
+ size: 32,
+ rd: from_reg,
+ mem,
+ lane_imm: 0,
+ },
+ types::F64 => Inst::VecStoreLane {
+ size: 64,
+ rd: from_reg,
+ mem,
+ lane_imm: 0,
+ },
+ _ if ty.is_vector() && ty.bits() == 128 => Inst::VecStore { rd: from_reg, mem },
_ => unimplemented!("gen_store({})", ty),
}
}
@@ -365,6 +437,9 @@ fn memarg_operands<F: Fn(VReg) -> VReg>(memarg: &MemArg, collector: &mut Operand
}
&MemArg::InitialSPOffset { .. } | &MemArg::NominalSPOffset { .. } => {}
}
+ // mem_finalize might require %r1 to hold (part of) the address.
+ // Conservatively assume this will always be necessary here.
+ collector.reg_early_def(writable_gpr(1));
}
fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCollector<'_, F>) {
@@ -579,13 +654,6 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandC
collector.reg_mod(rd);
collector.reg_use(rm);
}
- &Inst::MovToFpr32 { rd, rn }
- | &Inst::MovToFpr64 { rd, rn }
- | &Inst::MovFromFpr32 { rd, rn }
- | &Inst::MovFromFpr64 { rd, rn } => {
- collector.reg_def(rd);
- collector.reg_use(rn);
- }
&Inst::FpuRR { rd, rn, .. } => {
collector.reg_def(rd);
collector.reg_use(rn);
@@ -605,50 +673,158 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandC
collector.reg_use(rn);
collector.reg_use(rm);
}
- &Inst::FpuLoad32 { rd, ref mem, .. } => {
+ &Inst::LoadFpuConst32 { rd, .. } | &Inst::LoadFpuConst64 { rd, .. } => {
+ collector.reg_def(rd);
+ collector.reg_def(writable_gpr(1));
+ }
+ &Inst::FpuRound { rd, rn, .. } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ }
+ &Inst::VecRRR { rd, rn, rm, .. } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ collector.reg_use(rm);
+ }
+ &Inst::VecRR { rd, rn, .. } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ }
+ &Inst::VecShiftRR {
+ rd, rn, shift_reg, ..
+ } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ collector.reg_use(shift_reg);
+ }
+ &Inst::VecSelect { rd, rn, rm, ra, .. } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ collector.reg_use(rm);
+ collector.reg_use(ra);
+ }
+ &Inst::VecPermute { rd, rn, rm, ra, .. } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ collector.reg_use(rm);
+ collector.reg_use(ra);
+ }
+ &Inst::VecPermuteDWImm { rd, rn, rm, .. } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ collector.reg_use(rm);
+ }
+ &Inst::VecIntCmp { rd, rn, rm, .. } | &Inst::VecIntCmpS { rd, rn, rm, .. } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ collector.reg_use(rm);
+ }
+ &Inst::VecFloatCmp { rd, rn, rm, .. } | &Inst::VecFloatCmpS { rd, rn, rm, .. } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ collector.reg_use(rm);
+ }
+ &Inst::VecLoad { rd, ref mem, .. } => {
collector.reg_def(rd);
memarg_operands(mem, collector);
}
- &Inst::FpuLoad64 { rd, ref mem, .. } => {
+ &Inst::VecLoadRev { rd, ref mem, .. } => {
collector.reg_def(rd);
memarg_operands(mem, collector);
}
- &Inst::FpuStore32 { rd, ref mem, .. } => {
+ &Inst::VecStore { rd, ref mem, .. } => {
collector.reg_use(rd);
memarg_operands(mem, collector);
}
- &Inst::FpuStore64 { rd, ref mem, .. } => {
+ &Inst::VecStoreRev { rd, ref mem, .. } => {
collector.reg_use(rd);
memarg_operands(mem, collector);
}
- &Inst::FpuLoadRev32 { rd, ref mem, .. } => {
+ &Inst::VecLoadReplicate { rd, ref mem, .. } => {
+ collector.reg_def(rd);
+ memarg_operands(mem, collector);
+ }
+ &Inst::VecLoadReplicateRev { rd, ref mem, .. } => {
collector.reg_def(rd);
memarg_operands(mem, collector);
}
- &Inst::FpuLoadRev64 { rd, ref mem, .. } => {
+ &Inst::VecMov { rd, rn } => {
+ collector.reg_def(rd);
+ collector.reg_use(rn);
+ }
+ &Inst::VecCMov { rd, rm, .. } => {
+ collector.reg_mod(rd);
+ collector.reg_use(rm);
+ }
+ &Inst::MovToVec128 { rd, rn, rm } => {
collector.reg_def(rd);
+ collector.reg_use(rn);
+ collector.reg_use(rm);
+ }
+ &Inst::VecLoadConst { rd, .. } | &Inst::VecLoadConstReplicate { rd, .. } => {
+ collector.reg_def(rd);
+ collector.reg_def(writable_gpr(1));
+ }
+ &Inst::VecImmByteMask { rd, .. } => {
+ collector.reg_def(rd);
+ }
+ &Inst::VecImmBitMask { rd, .. } => {
+ collector.reg_def(rd);
+ }
+ &Inst::VecImmReplicate { rd, .. } => {
+ collector.reg_def(rd);
+ }
+ &Inst::VecLoadLane { rd, ref mem, .. } => {
+ collector.reg_mod(rd);
memarg_operands(mem, collector);
}
- &Inst::FpuStoreRev32 { rd, ref mem, .. } => {
+ &Inst::VecLoadLaneUndef { rd, ref mem, .. } => {
+ collector.reg_def(rd);
+ memarg_operands(mem, collector);
+ }
+ &Inst::VecStoreLaneRev { rd, ref mem, .. } => {
collector.reg_use(rd);
memarg_operands(mem, collector);
}
- &Inst::FpuStoreRev64 { rd, ref mem, .. } => {
+ &Inst::VecLoadLaneRevUndef { rd, ref mem, .. } => {
+ collector.reg_def(rd);
+ memarg_operands(mem, collector);
+ }
+ &Inst::VecStoreLane { rd, ref mem, .. } => {
collector.reg_use(rd);
memarg_operands(mem, collector);
}
- &Inst::LoadFpuConst32 { rd, .. } | &Inst::LoadFpuConst64 { rd, .. } => {
+ &Inst::VecLoadLaneRev { rd, ref mem, .. } => {
+ collector.reg_mod(rd);
+ memarg_operands(mem, collector);
+ }
+ &Inst::VecInsertLane {
+ rd, rn, lane_reg, ..
+ } => {
+ collector.reg_mod(rd);
+ collector.reg_use(rn);
+ collector.reg_use(lane_reg);
+ }
+ &Inst::VecInsertLaneUndef {
+ rd, rn, lane_reg, ..
+ } => {
collector.reg_def(rd);
+ collector.reg_use(rn);
+ collector.reg_use(lane_reg);
}
- &Inst::FpuRound { rd, rn, .. } => {
+ &Inst::VecExtractLane {
+ rd, rn, lane_reg, ..
+ } => {
collector.reg_def(rd);
collector.reg_use(rn);
+ collector.reg_use(lane_reg);
}
- &Inst::VecSelect { rd, rn, rm, ra, .. } => {
+ &Inst::VecInsertLaneImm { rd, .. } => {
+ collector.reg_def(rd);
+ }
+ &Inst::VecReplicateLane { rd, rn, .. } => {
collector.reg_def(rd);
collector.reg_use(rn);
- collector.reg_use(rm);
- collector.reg_use(ra);
}
&Inst::Extend { rd, rn, .. } => {
collector.reg_def(rd);
@@ -682,9 +858,11 @@ fn s390x_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandC
&Inst::TrapIf { .. } => {}
&Inst::JTSequence { ridx, .. } => {
collector.reg_use(ridx);
+ collector.reg_early_def(writable_gpr(1));
}
&Inst::LoadExtNameFar { rd, .. } => {
collector.reg_def(rd);
+ collector.reg_def(writable_gpr(1));
}
&Inst::LoadAddr { rd, ref mem } => {
collector.reg_def(rd);
@@ -720,6 +898,7 @@ impl MachInst for Inst {
&Inst::Mov64 { rd, rm } => Some((rd, rm)),
&Inst::FpuMove32 { rd, rn } => Some((rd, rn)),
&Inst::FpuMove64 { rd, rn } => Some((rd, rn)),
+ &Inst::VecMov { rd, rn } => Some((rd, rn)),
_ => None,
}
}
@@ -732,6 +911,21 @@ impl MachInst for Inst {
}
}
+ fn is_included_in_clobbers(&self) -> bool {
+ // We exclude call instructions from the clobber-set when they are calls
+ // from caller to callee with the same ABI. Such calls cannot possibly
+ // force any new registers to be saved in the prologue, because anything
+ // that the callee clobbers, the caller is also allowed to clobber. This
+ // both saves work and enables us to more precisely follow the
+ // half-caller-save, half-callee-save SysV ABI for some vector
+ // registers.
+ match self {
+ &Inst::Call { ref info, .. } => info.caller_callconv != info.callee_callconv,
+ &Inst::CallInd { ref info, .. } => info.caller_callconv != info.callee_callconv,
+ _ => true,
+ }
+ }
+
fn is_term(&self) -> MachTerminator {
match self {
&Inst::Ret { .. } | &Inst::EpiloguePlaceholder => MachTerminator::Ret,
@@ -761,11 +955,13 @@ impl MachInst for Inst {
}
fn gen_move(to_reg: Writable<Reg>, from_reg: Reg, ty: Type) -> Inst {
- assert!(ty.bits() <= 64);
+ assert!(ty.bits() <= 128);
if ty.bits() <= 32 {
Inst::mov32(to_reg, from_reg)
- } else {
+ } else if ty.bits() <= 64 {
Inst::mov64(to_reg, from_reg)
+ } else {
+ Inst::mov128(to_reg, from_reg)
}
}
@@ -778,11 +974,18 @@ impl MachInst for Inst {
let to_reg = to_regs
.only_reg()
.expect("multi-reg values not supported yet");
- let value = value as u64;
match ty {
+ _ if ty.is_vector() && ty.bits() == 128 => {
+ let mut ret = SmallVec::new();
+ ret.push(Inst::load_vec_constant(to_reg, value));
+ ret
+ }
types::F64 => {
let mut ret = SmallVec::new();
- ret.push(Inst::load_fp_constant64(to_reg, f64::from_bits(value)));
+ ret.push(Inst::load_fp_constant64(
+ to_reg,
+ f64::from_bits(value as u64),
+ ));
ret
}
types::F32 => {
@@ -793,7 +996,7 @@ impl MachInst for Inst {
));
ret
}
- types::I64 | types::B64 | types::R64 => Inst::load_constant64(to_reg, value),
+ types::I64 | types::B64 | types::R64 => Inst::load_constant64(to_reg, value as u64),
types::B1
| types::I8
| types::B8
@@ -832,6 +1035,7 @@ impl MachInst for Inst {
types::F64 => Ok((&[RegClass::Float], &[types::F64])),
types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])),
types::B128 => Ok((&[RegClass::Int, RegClass::Int], &[types::B64, types::B64])),
+ _ if ty.is_vector() && ty.bits() == 128 => Ok((&[RegClass::Float], &[types::I8X16])),
// FIXME: We don't really have IFLAGS, but need to allow it here
// for now to support the SelectifSpectreGuard instruction.
types::IFLAGS => Ok((&[RegClass::Int], &[types::I64])),
@@ -845,7 +1049,7 @@ impl MachInst for Inst {
fn canonical_type_for_rc(rc: RegClass) -> Type {
match rc {
RegClass::Int => types::I64,
- RegClass::Float => types::F64,
+ RegClass::Float => types::I8X16,
}
}
@@ -1497,43 +1701,6 @@ impl Inst {
let mem = mem.pretty_print_default();
format!("{}{} {}, {}", mem_str, op.unwrap(), rd, mem)
}
- &Inst::FpuLoad32 { rd, ref mem }
- | &Inst::FpuLoad64 { rd, ref mem }
- | &Inst::FpuLoadRev32 { rd, ref mem }
- | &Inst::FpuLoadRev64 { rd, ref mem } => {
- let (opcode_rx, opcode_rxy, opcode_vrx) = match self {
- &Inst::FpuLoad32 { .. } => (Some("le"), Some("ley"), "vlef"),
- &Inst::FpuLoad64 { .. } => (Some("ld"), Some("ldy"), "vleg"),
- &Inst::FpuLoadRev32 { .. } => (None, None, "vlebrf"),
- &Inst::FpuLoadRev64 { .. } => (None, None, "vlebrg"),
- _ => unreachable!(),
- };
-
- let (rd, rd_fpr) = pretty_print_fpr(rd.to_reg(), allocs);
- let mem = mem.with_allocs(allocs);
- if rd_fpr.is_some() && opcode_rx.is_some() {
- let (mem_str, mem) =
- mem_finalize_for_show(&mem, state, true, true, false, true);
- let op = match &mem {
- &MemArg::BXD12 { .. } => opcode_rx,
- &MemArg::BXD20 { .. } => opcode_rxy,
- _ => unreachable!(),
- };
- let mem = mem.pretty_print_default();
- format!("{}{} {}, {}", mem_str, op.unwrap(), rd_fpr.unwrap(), mem)
- } else {
- let (mem_str, mem) =
- mem_finalize_for_show(&mem, state, true, false, false, true);
- let mem = mem.pretty_print_default();
- format!(
- "{}{} {}, {}, 0",
- mem_str,
- opcode_vrx,
- rd_fpr.unwrap_or(rd),
- mem
- )
- }
- }
&Inst::Store8 { rd, ref mem }
| &Inst::Store16 { rd, ref mem }
| &Inst::Store32 { rd, ref mem }
@@ -1599,43 +1766,6 @@ impl Inst {
format!("{}{} {}, {}", mem_str, op, mem, imm)
}
- &Inst::FpuStore32 { rd, ref mem }
- | &Inst::FpuStore64 { rd, ref mem }
- | &Inst::FpuStoreRev32 { rd, ref mem }
- | &Inst::FpuStoreRev64 { rd, ref mem } => {
- let (opcode_rx, opcode_rxy, opcode_vrx) = match self {
- &Inst::FpuStore32 { .. } => (Some("ste"), Some("stey"), "vstef"),
- &Inst::FpuStore64 { .. } => (Some("std"), Some("stdy"), "vsteg"),
- &Inst::FpuStoreRev32 { .. } => (None, None, "vstebrf"),
- &Inst::FpuStoreRev64 { .. } => (None, None, "vstebrg"),
- _ => unreachable!(),
- };
-
- let (rd, rd_fpr) = pretty_print_fpr(rd, allocs);
- let mem = mem.with_allocs(allocs);
- if rd_fpr.is_some() && opcode_rx.is_some() {
- let (mem_str, mem) =
- mem_finalize_for_show(&mem, state, true, true, false, true);
- let op = match &mem {
- &MemArg::BXD12 { .. } => opcode_rx,
- &MemArg::BXD20 { .. } => opcode_rxy,
- _ => unreachable!(),
- };
- let mem = mem.pretty_print_default();
- format!("{}{} {}, {}", mem_str, op.unwrap(), rd_fpr.unwrap(), mem)
- } else {
- let (mem_str, mem) =
- mem_finalize_for_show(&mem, state, true, false, false, true);
- let mem = mem.pretty_print_default();
- format!(
- "{}{} {}, {}, 0",
- mem_str,
- opcode_vrx,
- rd_fpr.unwrap_or(rd),
- mem
- )
- }
- }
&Inst::LoadMultiple64 { rt, rt2, ref mem } => {
let mem = mem.with_allocs(allocs);
let (mem_str, mem) = mem_finalize_for_show(&mem, state, false, true, false, false);
@@ -1780,69 +1910,77 @@ impl Inst {
format!("j{} 10 ; vlr {}, {}", cond, rd, rm)
}
}
- &Inst::MovToFpr32 { rd, rn } => {
- let rd = pretty_print_reg(rd.to_reg(), allocs);
- let rn = pretty_print_reg(rn, allocs);
- format!("vlvgf {}, {}, 0", rd, rn)
- }
- &Inst::MovToFpr64 { rd, rn } => {
- let (rd, rd_fpr) = pretty_print_fpr(rd.to_reg(), allocs);
- let rn = pretty_print_reg(rn, allocs);
- if rd_fpr.is_some() {
- format!("ldgr {}, {}", rd_fpr.unwrap(), rn)
- } else {
- format!("vlvgg {}, {}, 0", rd, rn)
- }
- }
- &Inst::MovFromFpr32 { rd, rn } => {
- let rd = pretty_print_reg(rd.to_reg(), allocs);
- let rn = pretty_print_reg(rn, allocs);
- format!("vlgvf {}, {}, 0", rd, rn)
- }
- &Inst::MovFromFpr64 { rd, rn } => {
- let rd = pretty_print_reg(rd.to_reg(), allocs);
- let (rn, rn_fpr) = pretty_print_fpr(rn, allocs);
- if rn_fpr.is_some() {
- format!("lgdr {}, {}", rd, rn_fpr.unwrap())
- } else {
- format!("vlgvg {}, {}, 0", rd, rn)
- }
- }
&Inst::FpuRR { fpu_op, rd, rn } => {
let (op, op_fpr) = match fpu_op {
- FPUOp1::Abs32 => ("wflpsb", "lpebr"),
- FPUOp1::Abs64 => ("wflpdb", "lpdbr"),
- FPUOp1::Neg32 => ("wflcsb", "lcebr"),
- FPUOp1::Neg64 => ("wflcdb", "lcdbr"),
- FPUOp1::NegAbs32 => ("wflnsb", "lnebr"),
- FPUOp1::NegAbs64 => ("wflndb", "lndbr"),
- FPUOp1::Sqrt32 => ("wfsqsb", "sqebr"),
- FPUOp1::Sqrt64 => ("wfsqdb", "sqdbr"),
- FPUOp1::Cvt32To64 => ("wldeb", "ldebr"),
+ FPUOp1::Abs32 => ("wflpsb", Some("lpebr")),
+ FPUOp1::Abs64 => ("wflpdb", Some("lpdbr")),
+ FPUOp1::Abs32x4 => ("vflpsb", None),
+ FPUOp1::Abs64x2 => ("vflpdb", None),
+ FPUOp1::Neg32 => ("wflcsb", Some("lcebr")),
+ FPUOp1::Neg64 => ("wflcdb", Some("lcdbr")),
+ FPUOp1::Neg32x4 => ("vflcsb", None),
+ FPUOp1::Neg64x2 => ("vflcdb", None),
+ FPUOp1::NegAbs32 => ("wflnsb", Some("lnebr")),
+ FPUOp1::NegAbs64 => ("wflndb", Some("lndbr")),
+ FPUOp1::NegAbs32x4 => ("vflnsb", None),
+ FPUOp1::NegAbs64x2 => ("vflndb", None),
+ FPUOp1::Sqrt32 => ("wfsqsb", Some("sqebr")),
+ FPUOp1::Sqrt64 => ("wfsqdb", Some("sqdbr")),
+ FPUOp1::Sqrt32x4 => ("vfsqsb", None),
+ FPUOp1::Sqrt64x2 => ("vfsqdb", None),
+ FPUOp1::Cvt32To64 => ("wldeb", Some("ldebr")),
+ FPUOp1::Cvt32x4To64x2 => ("vldeb", None),
};
let (rd, rd_fpr) = pretty_print_fpr(rd.to_reg(), allocs);
let (rn, rn_fpr) = pretty_print_fpr(rn, allocs);
- if rd_fpr.is_some() && rn_fpr.is_some() {
- format!("{} {}, {}", op_fpr, rd_fpr.unwrap(), rn_fpr.unwrap())
- } else {
+ if op_fpr.is_some() && rd_fpr.is_some() && rn_fpr.is_some() {
+ format!(
+ "{} {}, {}",
+ op_fpr.unwrap(),
+ rd_fpr.unwrap(),
+ rn_fpr.unwrap()
+ )
+ } else if op.starts_with('w') {
format!("{} {}, {}", op, rd_fpr.unwrap_or(rd), rn_fpr.unwrap_or(rn))
+ } else {
+ format!("{} {}, {}", op, rd, rn)
}
}
&Inst::FpuRRR { fpu_op, rd, rn, rm } => {
let (op, opt_m6, op_fpr) = match fpu_op {
FPUOp2::Add32 => ("wfasb", "", Some("aebr")),
FPUOp2::Add64 => ("wfadb", "", Some("adbr")),
+ FPUOp2::Add32x4 => ("vfasb", "", None),
+ FPUOp2::Add64x2 => ("vfadb", "", None),
FPUOp2::Sub32 => ("wfssb", "", Some("sebr")),
FPUOp2::Sub64 => ("wfsdb", "", Some("sdbr")),
+ FPUOp2::Sub32x4 => ("vfssb", "", None),
+ FPUOp2::Sub64x2 => ("vfsdb", "", None),
FPUOp2::Mul32 => ("wfmsb", "", Some("meebr")),
FPUOp2::Mul64 => ("wfmdb", "", Some("mdbr")),
+ FPUOp2::Mul32x4 => ("vfmsb", "", None),
+ FPUOp2::Mul64x2 => ("vfmdb", "", None),
FPUOp2::Div32 => ("wfdsb", "", Some("debr")),
FPUOp2::Div64 => ("wfddb", "", Some("ddbr")),
+ FPUOp2::Div32x4 => ("vfdsb", "", None),
+ FPUOp2::Div64x2 => ("vfddb", "", None),
FPUOp2::Max32 => ("wfmaxsb", ", 1", None),
FPUOp2::Max64 => ("wfmaxdb", ", 1", None),
+ FPUOp2::Max32x4 => ("vfmaxsb", ", 1", None),
+ FPUOp2::Max64x2 => ("vfmaxdb", ", 1", None),
FPUOp2::Min32 => ("wfminsb", ", 1", None),
FPUOp2::Min64 => ("wfmindb", ", 1", None),
+ FPUOp2::Min32x4 => ("vfminsb", ", 1", None),
+ FPUOp2::Min64x2 => ("vfmindb", ", 1", None),
+ FPUOp2::MaxPseudo32 => ("wfmaxsb", ", 3", None),
+ FPUOp2::MaxPseudo64 => ("wfmaxdb", ", 3", None),
+ FPUOp2::MaxPseudo32x4 => ("vfmaxsb", ", 3", None),
+ FPUOp2::MaxPseudo64x2 => ("vfmaxdb", ", 3", None),
+ FPUOp2::MinPseudo32 => ("wfminsb", ", 3", None),
+ FPUOp2::MinPseudo64 => ("wfmindb", ", 3", None),
+ FPUOp2::MinPseudo32x4 => ("vfminsb", ", 3", None),
+ FPUOp2::MinPseudo64x2 => ("vfmindb", ", 3", None),
};
let (rd, rd_fpr) = pretty_print_fpr(rd.to_reg(), allocs);
@@ -1855,7 +1993,7 @@ impl Inst {
rd_fpr.unwrap(),
rm_fpr.unwrap()
)
- } else {
+ } else if op.starts_with('w') {
format!(
"{} {}, {}, {}{}",
op,
@@ -1864,6 +2002,8 @@ impl Inst {
rm_fpr.unwrap_or(rm),
opt_m6
)
+ } else {
+ format!("{} {}, {}, {}{}", op, rd, rn, rm, opt_m6)
}
}
&Inst::FpuRRRR {
@@ -1874,25 +2014,34 @@ impl Inst {
ra,
} => {
let (op, op_fpr) = match fpu_op {
- FPUOp3::MAdd32 => ("wfmasb", "maebr"),
- FPUOp3::MAdd64 => ("wfmadb", "madbr"),
- FPUOp3::MSub32 => ("wfmssb", "msebr"),
- FPUOp3::MSub64 => ("wfmsdb", "msdbr"),
+ FPUOp3::MAdd32 => ("wfmasb", Some("maebr")),
+ FPUOp3::MAdd64 => ("wfmadb", Some("madbr")),
+ FPUOp3::MAdd32x4 => ("vfmasb", None),
+ FPUOp3::MAdd64x2 => ("vfmadb", None),
+ FPUOp3::MSub32 => ("wfmssb", Some("msebr")),
+ FPUOp3::MSub64 => ("wfmsdb", Some("msdbr")),
+ FPUOp3::MSub32x4 => ("vfmssb", None),
+ FPUOp3::MSub64x2 => ("vfmsdb", None),
};
let (rd, rd_fpr) = pretty_print_fpr(rd.to_reg(), allocs);
let (rn, rn_fpr) = pretty_print_fpr(rn, allocs);
let (rm, rm_fpr) = pretty_print_fpr(rm, allocs);
let (ra, ra_fpr) = pretty_print_fpr(ra, allocs);
- if rd == ra && rd_fpr.is_some() && rn_fpr.is_some() && rm_fpr.is_some() {
+ if op_fpr.is_some()
+ && rd == ra
+ && rd_fpr.is_some()
+ && rn_fpr.is_some()
+ && rm_fpr.is_some()
+ {
format!(
"{} {}, {}, {}",
- op_fpr,
+ op_fpr.unwrap(),
rd_fpr.unwrap(),
rn_fpr.unwrap(),
rm_fpr.unwrap()
)
- } else {
+ } else if op.starts_with('w') {
format!(
"{} {}, {}, {}, {}",
op,
@@ -1901,6 +2050,8 @@ impl Inst {
rm_fpr.unwrap_or(rm),
ra_fpr.unwrap_or(ra)
)
+ } else {
+ format!("{} {}, {}, {}, {}", op, rd, rn, rm, ra)
}
}
&Inst::FpuCmp32 { rn, rm } => {
@@ -1975,16 +2126,27 @@ impl Inst {
};
let (opcode, opcode_fpr) = match op {
FpuRoundOp::Cvt64To32 => ("wledb", Some("ledbra")),
+ FpuRoundOp::Cvt64x2To32x4 => ("vledb", None),
FpuRoundOp::Round32 => ("wfisb", Some("fiebr")),
FpuRoundOp::Round64 => ("wfidb", Some("fidbr")),
+ FpuRoundOp::Round32x4 => ("vfisb", None),
+ FpuRoundOp::Round64x2 => ("vfidb", None),
FpuRoundOp::ToSInt32 => ("wcfeb", None),
FpuRoundOp::ToSInt64 => ("wcgdb", None),
FpuRoundOp::ToUInt32 => ("wclfeb", None),
FpuRoundOp::ToUInt64 => ("wclgdb", None),
+ FpuRoundOp::ToSInt32x4 => ("vcfeb", None),
+ FpuRoundOp::ToSInt64x2 => ("vcgdb", None),
+ FpuRoundOp::ToUInt32x4 => ("vclfeb", None),
+ FpuRoundOp::ToUInt64x2 => ("vclgdb", None),
FpuRoundOp::FromSInt32 => ("wcefb", None),
FpuRoundOp::FromSInt64 => ("wcdgb", None),
FpuRoundOp::FromUInt32 => ("wcelfb", None),
FpuRoundOp::FromUInt64 => ("wcdlgb", None),
+ FpuRoundOp::FromSInt32x4 => ("vcefb", None),
+ FpuRoundOp::FromSInt64x2 => ("vcdgb", None),
+ FpuRoundOp::FromUInt32x4 => ("vcelfb", None),
+ FpuRoundOp::FromUInt64x2 => ("vcdlgb", None),
};
let (rd, rd_fpr) = pretty_print_fpr(rd.to_reg(), allocs);
@@ -1997,7 +2159,7 @@ impl Inst {
rn_fpr.unwrap(),
mode
)
- } else {
+ } else if opcode.starts_with('w') {
format!(
"{} {}, {}, 0, {}",
opcode,
@@ -2005,8 +2167,168 @@ impl Inst {
rn_fpr.unwrap_or(rn),
mode
)
+ } else {
+ format!("{} {}, {}, 0, {}", opcode, rd, rn, mode)
}
}
+ &Inst::VecRRR { op, rd, rn, rm } => {
+ let op = match op {
+ VecBinaryOp::Add8x16 => "vab",
+ VecBinaryOp::Add16x8 => "vah",
+ VecBinaryOp::Add32x4 => "vaf",
+ VecBinaryOp::Add64x2 => "vag",
+ VecBinaryOp::Sub8x16 => "vsb",
+ VecBinaryOp::Sub16x8 => "vsh",
+ VecBinaryOp::Sub32x4 => "vsf",
+ VecBinaryOp::Sub64x2 => "vsg",
+ VecBinaryOp::Mul8x16 => "vmlb",
+ VecBinaryOp::Mul16x8 => "vmlhw",
+ VecBinaryOp::Mul32x4 => "vmlf",
+ VecBinaryOp::UMulHi8x16 => "vmlhb",
+ VecBinaryOp::UMulHi16x8 => "vmlhh",
+ VecBinaryOp::UMulHi32x4 => "vmlhf",
+ VecBinaryOp::SMulHi8x16 => "vmhb",
+ VecBinaryOp::SMulHi16x8 => "vmhh",
+ VecBinaryOp::SMulHi32x4 => "vmhf",
+ VecBinaryOp::UMulEven8x16 => "vmleb",
+ VecBinaryOp::UMulEven16x8 => "vmleh",
+ VecBinaryOp::UMulEven32x4 => "vmlef",
+ VecBinaryOp::SMulEven8x16 => "vmeb",
+ VecBinaryOp::SMulEven16x8 => "vmeh",
+ VecBinaryOp::SMulEven32x4 => "vmef",
+ VecBinaryOp::UMulOdd8x16 => "vmlob",
+ VecBinaryOp::UMulOdd16x8 => "vmloh",
+ VecBinaryOp::UMulOdd32x4 => "vmlof",
+ VecBinaryOp::SMulOdd8x16 => "vmob",
+ VecBinaryOp::SMulOdd16x8 => "vmoh",
+ VecBinaryOp::SMulOdd32x4 => "vmof",
+ VecBinaryOp::UMax8x16 => "vmxlb",
+ VecBinaryOp::UMax16x8 => "vmxlh",
+ VecBinaryOp::UMax32x4 => "vmxlf",
+ VecBinaryOp::UMax64x2 => "vmxlg",
+ VecBinaryOp::SMax8x16 => "vmxb",
+ VecBinaryOp::SMax16x8 => "vmxh",
+ VecBinaryOp::SMax32x4 => "vmxf",
+ VecBinaryOp::SMax64x2 => "vmxg",
+ VecBinaryOp::UMin8x16 => "vmnlb",
+ VecBinaryOp::UMin16x8 => "vmnlh",
+ VecBinaryOp::UMin32x4 => "vmnlf",
+ VecBinaryOp::UMin64x2 => "vmnlg",
+ VecBinaryOp::SMin8x16 => "vmnb",
+ VecBinaryOp::SMin16x8 => "vmnh",
+ VecBinaryOp::SMin32x4 => "vmnf",
+ VecBinaryOp::SMin64x2 => "vmng",
+ VecBinaryOp::UAvg8x16 => "vavglb",
+ VecBinaryOp::UAvg16x8 => "vavglh",
+ VecBinaryOp::UAvg32x4 => "vavglf",
+ VecBinaryOp::UAvg64x2 => "vavglg",
+ VecBinaryOp::SAvg8x16 => "vavgb",
+ VecBinaryOp::SAvg16x8 => "vavgh",
+ VecBinaryOp::SAvg32x4 => "vavgf",
+ VecBinaryOp::SAvg64x2 => "vavgg",
+ VecBinaryOp::And128 => "vn",
+ VecBinaryOp::Orr128 => "vo",
+ VecBinaryOp::Xor128 => "vx",
+ VecBinaryOp::NotAnd128 => "vnn",
+ VecBinaryOp::NotOrr128 => "vno",
+ VecBinaryOp::NotXor128 => "vnx",
+ VecBinaryOp::AndNot128 => "vnc",
+ VecBinaryOp::OrrNot128 => "voc",
+ VecBinaryOp::BitPermute128 => "vbperm",
+ VecBinaryOp::LShLByByte128 => "vslb",
+ VecBinaryOp::LShRByByte128 => "vsrlb",
+ VecBinaryOp::AShRByByte128 => "vsrab",
+ VecBinaryOp::LShLByBit128 => "vsl",
+ VecBinaryOp::LShRByBit128 => "vsrl",
+ VecBinaryOp::AShRByBit128 => "vsra",
+ VecBinaryOp::Pack16x8 => "vpkh",
+ VecBinaryOp::Pack32x4 => "vpkf",
+ VecBinaryOp::Pack64x2 => "vpkg",
+ VecBinaryOp::PackUSat16x8 => "vpklsh",
+ VecBinaryOp::PackUSat32x4 => "vpklsf",
+ VecBinaryOp::PackUSat64x2 => "vpklsg",
+ VecBinaryOp::PackSSat16x8 => "vpksh",
+ VecBinaryOp::PackSSat32x4 => "vpksf",
+ VecBinaryOp::PackSSat64x2 => "vpksg",
+ VecBinaryOp::MergeLow8x16 => "vmrlb",
+ VecBinaryOp::MergeLow16x8 => "vmrlh",
+ VecBinaryOp::MergeLow32x4 => "vmrlf",
+ VecBinaryOp::MergeLow64x2 => "vmrlg",
+ VecBinaryOp::MergeHigh8x16 => "vmrhb",
+ VecBinaryOp::MergeHigh16x8 => "vmrhh",
+ VecBinaryOp::MergeHigh32x4 => "vmrhf",
+ VecBinaryOp::MergeHigh64x2 => "vmrhg",
+ };
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ let rm = pretty_print_reg(rm, allocs);
+ format!("{} {}, {}, {}", op, rd, rn, rm)
+ }
+ &Inst::VecRR { op, rd, rn } => {
+ let op = match op {
+ VecUnaryOp::Abs8x16 => "vlpb",
+ VecUnaryOp::Abs16x8 => "vlph",
+ VecUnaryOp::Abs32x4 => "vlpf",
+ VecUnaryOp::Abs64x2 => "vlpg",
+ VecUnaryOp::Neg8x16 => "vlcb",
+ VecUnaryOp::Neg16x8 => "vlch",
+ VecUnaryOp::Neg32x4 => "vlcf",
+ VecUnaryOp::Neg64x2 => "vlcg",
+ VecUnaryOp::Popcnt8x16 => "vpopctb",
+ VecUnaryOp::Popcnt16x8 => "vpopcth",
+ VecUnaryOp::Popcnt32x4 => "vpopctf",
+ VecUnaryOp::Popcnt64x2 => "vpopctg",
+ VecUnaryOp::UnpackULow8x16 => "vupllb",
+ VecUnaryOp::UnpackULow16x8 => "vupllh",
+ VecUnaryOp::UnpackULow32x4 => "vupllf",
+ VecUnaryOp::UnpackUHigh8x16 => "vuplhb",
+ VecUnaryOp::UnpackUHigh16x8 => "vuplhh",
+ VecUnaryOp::UnpackUHigh32x4 => "vuplhf",
+ VecUnaryOp::UnpackSLow8x16 => "vuplb",
+ VecUnaryOp::UnpackSLow16x8 => "vuplh",
+ VecUnaryOp::UnpackSLow32x4 => "vuplf",
+ VecUnaryOp::UnpackSHigh8x16 => "vuphb",
+ VecUnaryOp::UnpackSHigh16x8 => "vuphh",
+ VecUnaryOp::UnpackSHigh32x4 => "vuphf",
+ };
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ format!("{} {}, {}", op, rd, rn)
+ }
+ &Inst::VecShiftRR {
+ shift_op,
+ rd,
+ rn,
+ shift_imm,
+ shift_reg,
+ } => {
+ let op = match shift_op {
+ VecShiftOp::RotL8x16 => "verllb",
+ VecShiftOp::RotL16x8 => "verllh",
+ VecShiftOp::RotL32x4 => "verllf",
+ VecShiftOp::RotL64x2 => "verllg",
+ VecShiftOp::LShL8x16 => "veslb",
+ VecShiftOp::LShL16x8 => "veslh",
+ VecShiftOp::LShL32x4 => "veslf",
+ VecShiftOp::LShL64x2 => "veslg",
+ VecShiftOp::LShR8x16 => "vesrlb",
+ VecShiftOp::LShR16x8 => "vesrlh",
+ VecShiftOp::LShR32x4 => "vesrlf",
+ VecShiftOp::LShR64x2 => "vesrlg",
+ VecShiftOp::AShR8x16 => "vesrab",
+ VecShiftOp::AShR16x8 => "vesrah",
+ VecShiftOp::AShR32x4 => "vesraf",
+ VecShiftOp::AShR64x2 => "vesrag",
+ };
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ let shift_reg = if shift_reg != zero_reg() {
+ format!("({})", pretty_print_reg(shift_reg, allocs))
+ } else {
+ "".to_string()
+ };
+ format!("{} {}, {}, {}{}", op, rd, rn, shift_imm, shift_reg)
+ }
&Inst::VecSelect { rd, rn, rm, ra } => {
let rd = pretty_print_reg(rd.to_reg(), allocs);
let rn = pretty_print_reg(rn, allocs);
@@ -2014,6 +2336,409 @@ impl Inst {
let ra = pretty_print_reg(ra, allocs);
format!("vsel {}, {}, {}, {}", rd, rn, rm, ra)
}
+ &Inst::VecPermute { rd, rn, rm, ra } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ let rm = pretty_print_reg(rm, allocs);
+ let ra = pretty_print_reg(ra, allocs);
+ format!("vperm {}, {}, {}, {}", rd, rn, rm, ra)
+ }
+ &Inst::VecPermuteDWImm {
+ rd,
+ rn,
+ rm,
+ idx1,
+ idx2,
+ } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ let rm = pretty_print_reg(rm, allocs);
+ let m4 = (idx1 & 1) * 4 + (idx2 & 1);
+ format!("vpdi {}, {}, {}, {}", rd, rn, rm, m4)
+ }
+ &Inst::VecIntCmp { op, rd, rn, rm } | &Inst::VecIntCmpS { op, rd, rn, rm } => {
+ let op = match op {
+ VecIntCmpOp::CmpEq8x16 => "vceqb",
+ VecIntCmpOp::CmpEq16x8 => "vceqh",
+ VecIntCmpOp::CmpEq32x4 => "vceqf",
+ VecIntCmpOp::CmpEq64x2 => "vceqg",
+ VecIntCmpOp::SCmpHi8x16 => "vchb",
+ VecIntCmpOp::SCmpHi16x8 => "vchh",
+ VecIntCmpOp::SCmpHi32x4 => "vchf",
+ VecIntCmpOp::SCmpHi64x2 => "vchg",
+ VecIntCmpOp::UCmpHi8x16 => "vchlb",
+ VecIntCmpOp::UCmpHi16x8 => "vchlh",
+ VecIntCmpOp::UCmpHi32x4 => "vchlf",
+ VecIntCmpOp::UCmpHi64x2 => "vchlg",
+ };
+ let s = match self {
+ &Inst::VecIntCmp { .. } => "",
+ &Inst::VecIntCmpS { .. } => "s",
+ _ => unreachable!(),
+ };
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ let rm = pretty_print_reg(rm, allocs);
+ format!("{}{} {}, {}, {}", op, s, rd, rn, rm)
+ }
+ &Inst::VecFloatCmp { op, rd, rn, rm } | &Inst::VecFloatCmpS { op, rd, rn, rm } => {
+ let op = match op {
+ VecFloatCmpOp::CmpEq32x4 => "vfcesb",
+ VecFloatCmpOp::CmpEq64x2 => "vfcedb",
+ VecFloatCmpOp::CmpHi32x4 => "vfchsb",
+ VecFloatCmpOp::CmpHi64x2 => "vfchdb",
+ VecFloatCmpOp::CmpHiEq32x4 => "vfchesb",
+ VecFloatCmpOp::CmpHiEq64x2 => "vfchedb",
+ };
+ let s = match self {
+ &Inst::VecFloatCmp { .. } => "",
+ &Inst::VecFloatCmpS { .. } => "s",
+ _ => unreachable!(),
+ };
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ let rm = pretty_print_reg(rm, allocs);
+ format!("{}{} {}, {}, {}", op, s, rd, rn, rm)
+ }
+ &Inst::VecLoad { rd, ref mem } | &Inst::VecLoadRev { rd, ref mem } => {
+ let opcode = match self {
+ &Inst::VecLoad { .. } => "vl",
+ &Inst::VecLoadRev { .. } => "vlbrq",
+ _ => unreachable!(),
+ };
+
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let mem = mem.with_allocs(allocs);
+ let (mem_str, mem) = mem_finalize_for_show(&mem, state, true, false, false, true);
+ let mem = mem.pretty_print_default();
+ format!("{}{} {}, {}", mem_str, opcode, rd, mem)
+ }
+ &Inst::VecStore { rd, ref mem } | &Inst::VecStoreRev { rd, ref mem } => {
+ let opcode = match self {
+ &Inst::VecStore { .. } => "vst",
+ &Inst::VecStoreRev { .. } => "vstbrq",
+ _ => unreachable!(),
+ };
+
+ let rd = pretty_print_reg(rd, allocs);
+ let mem = mem.with_allocs(allocs);
+ let (mem_str, mem) = mem_finalize_for_show(&mem, state, true, false, false, true);
+ let mem = mem.pretty_print_default();
+ format!("{}{} {}, {}", mem_str, opcode, rd, mem)
+ }
+ &Inst::VecLoadReplicate { size, rd, ref mem }
+ | &Inst::VecLoadReplicateRev { size, rd, ref mem } => {
+ let opcode = match (self, size) {
+ (&Inst::VecLoadReplicate { .. }, 8) => "vlrepb",
+ (&Inst::VecLoadReplicate { .. }, 16) => "vlreph",
+ (&Inst::VecLoadReplicate { .. }, 32) => "vlrepf",
+ (&Inst::VecLoadReplicate { .. }, 64) => "vlrepg",
+ (&Inst::VecLoadReplicateRev { .. }, 16) => "vlbrreph",
+ (&Inst::VecLoadReplicateRev { .. }, 32) => "vlbrrepf",
+ (&Inst::VecLoadReplicateRev { .. }, 64) => "vlbrrepg",
+ _ => unreachable!(),
+ };
+
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let mem = mem.with_allocs(allocs);
+ let (mem_str, mem) = mem_finalize_for_show(&mem, state, true, false, false, true);
+ let mem = mem.pretty_print_default();
+ format!("{}{} {}, {}", mem_str, opcode, rd, mem)
+ }
+ &Inst::VecMov { rd, rn } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ format!("vlr {}, {}", rd, rn)
+ }
+ &Inst::VecCMov { rd, cond, rm } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rm = pretty_print_reg(rm, allocs);
+ let cond = cond.invert().pretty_print_default();
+ format!("j{} 10 ; vlr {}, {}", cond, rd, rm)
+ }
+ &Inst::MovToVec128 { rd, rn, rm } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ let rm = pretty_print_reg(rm, allocs);
+ format!("vlvgp {}, {}, {}", rd, rn, rm)
+ }
+ &Inst::VecLoadConst { rd, const_data } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let tmp = pretty_print_reg(writable_spilltmp_reg().to_reg(), &mut empty_allocs);
+ format!(
+ "bras {}, 20 ; data.u128 0x{:032x} ; vl {}, 0({})",
+ tmp, const_data, rd, tmp
+ )
+ }
+ &Inst::VecLoadConstReplicate {
+ size,
+ rd,
+ const_data,
+ } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let tmp = pretty_print_reg(writable_spilltmp_reg().to_reg(), &mut empty_allocs);
+ let (opcode, data) = match size {
+ 32 => ("vlrepf", format!("0x{:08x}", const_data as u32)),
+ 64 => ("vlrepg", format!("0x{:016x}", const_data)),
+ _ => unreachable!(),
+ };
+ format!(
+ "bras {}, {} ; data.u{} {} ; {} {}, 0({})",
+ tmp,
+ 4 + size / 8,
+ size,
+ data,
+ opcode,
+ rd,
+ tmp
+ )
+ }
+ &Inst::VecImmByteMask { rd, mask } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ format!("vgbm {}, {}", rd, mask)
+ }
+ &Inst::VecImmBitMask {
+ size,
+ rd,
+ start_bit,
+ end_bit,
+ } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let op = match size {
+ 8 => "vgmb",
+ 16 => "vgmh",
+ 32 => "vgmf",
+ 64 => "vgmg",
+ _ => unreachable!(),
+ };
+ format!("{} {}, {}, {}", op, rd, start_bit, end_bit)
+ }
+ &Inst::VecImmReplicate { size, rd, imm } => {
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let op = match size {
+ 8 => "vrepib",
+ 16 => "vrepih",
+ 32 => "vrepif",
+ 64 => "vrepig",
+ _ => unreachable!(),
+ };
+ format!("{} {}, {}", op, rd, imm)
+ }
+ &Inst::VecLoadLane {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ }
+ | &Inst::VecLoadLaneRev {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ }
+ | &Inst::VecLoadLaneUndef {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ }
+ | &Inst::VecLoadLaneRevUndef {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ } => {
+ let (opcode_vrx, opcode_rx, opcode_rxy) = match (self, size) {
+ (&Inst::VecLoadLane { .. }, 8) => ("vleb", None, None),
+ (&Inst::VecLoadLane { .. }, 16) => ("vleh", None, None),
+ (&Inst::VecLoadLane { .. }, 32) => ("vlef", None, None),
+ (&Inst::VecLoadLane { .. }, 64) => ("vleg", None, None),
+ (&Inst::VecLoadLaneRev { .. }, 16) => ("vlebrh", None, None),
+ (&Inst::VecLoadLaneRev { .. }, 32) => ("vlebrf", None, None),
+ (&Inst::VecLoadLaneRev { .. }, 64) => ("vlebrg", None, None),
+ (&Inst::VecLoadLaneUndef { .. }, 8) => ("vleb", None, None),
+ (&Inst::VecLoadLaneUndef { .. }, 16) => ("vleh", None, None),
+ (&Inst::VecLoadLaneUndef { .. }, 32) => ("vlef", Some("le"), Some("ley")),
+ (&Inst::VecLoadLaneUndef { .. }, 64) => ("vleg", Some("ld"), Some("ldy")),
+ (&Inst::VecLoadLaneRevUndef { .. }, 16) => ("vlebrh", None, None),
+ (&Inst::VecLoadLaneRevUndef { .. }, 32) => ("vlebrf", None, None),
+ (&Inst::VecLoadLaneRevUndef { .. }, 64) => ("vlebrg", None, None),
+ _ => unreachable!(),
+ };
+
+ let (rd, rd_fpr) = pretty_print_fpr(rd.to_reg(), allocs);
+ let mem = mem.with_allocs(allocs);
+ if lane_imm == 0 && rd_fpr.is_some() && opcode_rx.is_some() {
+ let (mem_str, mem) =
+ mem_finalize_for_show(&mem, state, true, true, false, true);
+ let op = match &mem {
+ &MemArg::BXD12 { .. } => opcode_rx,
+ &MemArg::BXD20 { .. } => opcode_rxy,
+ _ => unreachable!(),
+ };
+ let mem = mem.pretty_print_default();
+ format!("{}{} {}, {}", mem_str, op.unwrap(), rd_fpr.unwrap(), mem)
+ } else {
+ let (mem_str, mem) =
+ mem_finalize_for_show(&mem, state, true, false, false, true);
+ let mem = mem.pretty_print_default();
+ format!("{}{} {}, {}, {}", mem_str, opcode_vrx, rd, mem, lane_imm)
+ }
+ }
+ &Inst::VecStoreLane {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ }
+ | &Inst::VecStoreLaneRev {
+ size,
+ rd,
+ ref mem,
+ lane_imm,
+ } => {
+ let (opcode_vrx, opcode_rx, opcode_rxy) = match (self, size) {
+ (&Inst::VecStoreLane { .. }, 8) => ("vsteb", None, None),
+ (&Inst::VecStoreLane { .. }, 16) => ("vsteh", None, None),
+ (&Inst::VecStoreLane { .. }, 32) => ("vstef", Some("ste"), Some("stey")),
+ (&Inst::VecStoreLane { .. }, 64) => ("vsteg", Some("std"), Some("stdy")),
+ (&Inst::VecStoreLaneRev { .. }, 16) => ("vstebrh", None, None),
+ (&Inst::VecStoreLaneRev { .. }, 32) => ("vstebrf", None, None),
+ (&Inst::VecStoreLaneRev { .. }, 64) => ("vstebrg", None, None),
+ _ => unreachable!(),
+ };
+
+ let (rd, rd_fpr) = pretty_print_fpr(rd, allocs);
+ let mem = mem.with_allocs(allocs);
+ if lane_imm == 0 && rd_fpr.is_some() && opcode_rx.is_some() {
+ let (mem_str, mem) =
+ mem_finalize_for_show(&mem, state, true, true, false, true);
+ let op = match &mem {
+ &MemArg::BXD12 { .. } => opcode_rx,
+ &MemArg::BXD20 { .. } => opcode_rxy,
+ _ => unreachable!(),
+ };
+ let mem = mem.pretty_print_default();
+ format!("{}{} {}, {}", mem_str, op.unwrap(), rd_fpr.unwrap(), mem)
+ } else {
+ let (mem_str, mem) =
+ mem_finalize_for_show(&mem, state, true, false, false, true);
+ let mem = mem.pretty_print_default();
+ format!("{}{} {}, {}, {}", mem_str, opcode_vrx, rd, mem, lane_imm,)
+ }
+ }
+ &Inst::VecInsertLane {
+ size,
+ rd,
+ rn,
+ lane_imm,
+ lane_reg,
+ } => {
+ let op = match size {
+ 8 => "vlvgb",
+ 16 => "vlvgh",
+ 32 => "vlvgf",
+ 64 => "vlvgg",
+ _ => unreachable!(),
+ };
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ let lane_reg = if lane_reg != zero_reg() {
+ format!("({})", pretty_print_reg(lane_reg, allocs))
+ } else {
+ "".to_string()
+ };
+ format!("{} {}, {}, {}{}", op, rd, rn, lane_imm, lane_reg)
+ }
+ &Inst::VecInsertLaneUndef {
+ size,
+ rd,
+ rn,
+ lane_imm,
+ lane_reg,
+ } => {
+ let (opcode_vrs, opcode_rre) = match size {
+ 8 => ("vlvgb", None),
+ 16 => ("vlvgh", None),
+ 32 => ("vlvgf", None),
+ 64 => ("vlvgg", Some("ldgr")),
+ _ => unreachable!(),
+ };
+ let (rd, rd_fpr) = pretty_print_fpr(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ let lane_reg = if lane_reg != zero_reg() {
+ format!("({})", pretty_print_reg(lane_reg, allocs))
+ } else {
+ "".to_string()
+ };
+ if opcode_rre.is_some() && lane_imm == 0 && lane_reg.is_empty() && rd_fpr.is_some()
+ {
+ format!("{} {}, {}", opcode_rre.unwrap(), rd_fpr.unwrap(), rn)
+ } else {
+ format!("{} {}, {}, {}{}", opcode_vrs, rd, rn, lane_imm, lane_reg)
+ }
+ }
+ &Inst::VecExtractLane {
+ size,
+ rd,
+ rn,
+ lane_imm,
+ lane_reg,
+ } => {
+ let (opcode_vrs, opcode_rre) = match size {
+ 8 => ("vlgvb", None),
+ 16 => ("vlgvh", None),
+ 32 => ("vlgvf", None),
+ 64 => ("vlgvg", Some("lgdr")),
+ _ => unreachable!(),
+ };
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let (rn, rn_fpr) = pretty_print_fpr(rn, allocs);
+ let lane_reg = if lane_reg != zero_reg() {
+ format!("({})", pretty_print_reg(lane_reg, allocs))
+ } else {
+ "".to_string()
+ };
+ if opcode_rre.is_some() && lane_imm == 0 && lane_reg.is_empty() && rn_fpr.is_some()
+ {
+ format!("{} {}, {}", opcode_rre.unwrap(), rd, rn_fpr.unwrap())
+ } else {
+ format!("{} {}, {}, {}{}", opcode_vrs, rd, rn, lane_imm, lane_reg)
+ }
+ }
+ &Inst::VecInsertLaneImm {
+ size,
+ rd,
+ imm,
+ lane_imm,
+ } => {
+ let op = match size {
+ 8 => "vleib",
+ 16 => "vleih",
+ 32 => "vleif",
+ 64 => "vleig",
+ _ => unreachable!(),
+ };
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ format!("{} {}, {}, {}", op, rd, imm, lane_imm)
+ }
+ &Inst::VecReplicateLane {
+ size,
+ rd,
+ rn,
+ lane_imm,
+ } => {
+ let op = match size {
+ 8 => "vrepb",
+ 16 => "vreph",
+ 32 => "vrepf",
+ 64 => "vrepg",
+ _ => unreachable!(),
+ };
+ let rd = pretty_print_reg(rd.to_reg(), allocs);
+ let rn = pretty_print_reg(rn, allocs);
+ format!("{} {}, {}, {}", op, rd, rn, lane_imm)
+ }
&Inst::Extend {
rd,
rn,
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
index 5dfc2ec3eca7..0685d7e653f3 100644
--- a/cranelift/codegen/src/isa/s390x/lower.isle
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -36,6 +36,12 @@
(imm $F64 x))
+;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (has_type ty (vconst (u128_from_constant x))))
+ (vec_imm ty x))
+
+
;;;; Rules for `null` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty (null)))
@@ -98,6 +104,39 @@
(rule (lower (has_type (fits_in_64 ty) (iadd (sinkable_sload32 x) y)))
(add_mem_sext32 ty y (sink_sload32 x)))
+;; Add two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (iadd x y)))
+ (vec_add ty x y))
+
+
+;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Add (saturate unsigned) two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (uadd_sat x y)))
+ (let ((sum Reg (vec_add ty x y)))
+ (vec_or ty sum (vec_cmphl ty x sum))))
+
+
+;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Add (saturate signed) two vector registers. $I64X2 not supported.
+(rule (lower (has_type (ty_vec128 ty) (sadd_sat x y)))
+ (vec_pack_ssat (vec_widen_type ty)
+ (vec_add (vec_widen_type ty) (vec_unpacks_high ty x)
+ (vec_unpacks_high ty y))
+ (vec_add (vec_widen_type ty) (vec_unpacks_low ty x)
+ (vec_unpacks_low ty y))))
+
+
+;;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Lane-wise integer pairwise addition for 8-/16/32-bit vector registers.
+(rule (lower (has_type ty @ (multi_lane bits _) (iadd_pairwise x y)))
+ (let ((size Reg (vec_imm_splat $I8X16 (u32_as_u64 bits))))
+ (vec_pack (vec_widen_type ty)
+ (vec_add ty y (vec_lshr_by_byte y size))
+ (vec_add ty x (vec_lshr_by_byte x size)))))
+
;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -129,6 +168,28 @@
(rule (lower (has_type (fits_in_64 ty) (isub x (sinkable_sload32 y))))
(sub_mem_sext32 ty x (sink_sload32 y)))
+;; Sub two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (isub x y)))
+ (vec_sub ty x y))
+
+
+;;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Add (saturate unsigned) two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (usub_sat x y)))
+ (vec_and ty (vec_sub ty x y) (vec_cmphl ty x y)))
+
+
+;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Add (saturate signed) two vector registers. $I64X2 not supported.
+(rule (lower (has_type (ty_vec128 ty) (ssub_sat x y)))
+ (vec_pack_ssat (vec_widen_type ty)
+ (vec_sub (vec_widen_type ty) (vec_unpacks_high ty x)
+ (vec_unpacks_high ty y))
+ (vec_sub (vec_widen_type ty) (vec_unpacks_low ty x)
+ (vec_unpacks_low ty y))))
+
;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -141,6 +202,10 @@
(rule (lower (has_type (fits_in_64 ty) (iabs (sext32_value x))))
(abs_reg_sext32 ty x))
+;; Absolute value of a vector register.
+(rule (lower (has_type (ty_vec128 ty) (iabs x)))
+ (vec_abs ty x))
+
;;;; Rules for `iadd_ifcout` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -193,6 +258,45 @@
(rule (lower (has_type (fits_in_64 ty) (ineg (sext32_value x))))
(neg_reg_sext32 ty x))
+;; Negate a vector register.
+(rule (lower (has_type (ty_vec128 ty) (ineg x)))
+ (vec_neg ty x))
+
+
+;;;; Rules for `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Unsigned maximum of two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (umax x y)))
+ (vec_umax ty x y))
+
+
+;;;; Rules for `umin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Unsigned minimum of two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (umin x y)))
+ (vec_umin ty x y))
+
+
+;;;; Rules for `imax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Signed maximum of two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (imax x y)))
+ (vec_smax ty x y))
+
+
+;;;; Rules for `imin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Signed minimum of two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (imin x y)))
+ (vec_smin ty x y))
+
+
+;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Unsigned average of two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (avg_round x y)))
+ (vec_uavg ty x y))
+
;;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -238,6 +342,24 @@
(rule (lower (has_type (fits_in_64 ty) (imul (sinkable_sload32 x) y)))
(mul_mem_sext32 ty y (sink_sload32 x)))
+;; Multiply two vector registers, using a helper.
+(decl vec_mul_impl (Type Reg Reg) Reg)
+(rule (lower (has_type (ty_vec128 ty) (imul x y)))
+ (vec_mul_impl ty x y))
+
+;; Multiply two vector registers - byte, halfword, and word.
+(rule (vec_mul_impl $I8X16 x y) (vec_mul $I8X16 x y))
+(rule (vec_mul_impl $I16X8 x y) (vec_mul $I16X8 x y))
+(rule (vec_mul_impl $I32X4 x y) (vec_mul $I32X4 x y))
+
+;; Multiply two vector registers - doubleword. Has to be scalarized.
+(rule (vec_mul_impl $I64X2 x y)
+ (mov_to_vec128 $I64X2
+ (mul_reg $I64 (vec_extract_lane $I64X2 x 0 (zero_reg))
+ (vec_extract_lane $I64X2 y 0 (zero_reg)))
+ (mul_reg $I64 (vec_extract_lane $I64X2 x 1 (zero_reg))
+ (vec_extract_lane $I64X2 y 1 (zero_reg)))))
+
;;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -260,6 +382,22 @@
(let ((pair RegPair (umul_wide x y)))
(copy_reg $I64 (regpair_hi pair))))
+;; Multiply high part unsigned, vector types with 8-, 16-, or 32-bit elements.
+(rule (lower (has_type $I8X16 (umulhi x y))) (vec_umulhi $I8X16 x y))
+(rule (lower (has_type $I16X8 (umulhi x y))) (vec_umulhi $I16X8 x y))
+(rule (lower (has_type $I32X4 (umulhi x y))) (vec_umulhi $I32X4 x y))
+
+;; Multiply high part unsigned, vector types with 64-bit elements.
+;; Has to be scalarized.
+(rule (lower (has_type $I64X2 (umulhi x y)))
+ (let ((pair_0 RegPair (umul_wide (vec_extract_lane $I64X2 x 0 (zero_reg))
+ (vec_extract_lane $I64X2 y 0 (zero_reg))))
+ (res_0 Reg (copy_reg $I64 (regpair_hi pair_0)))
+ (pair_1 RegPair (umul_wide (vec_extract_lane $I64X2 x 1 (zero_reg))
+ (vec_extract_lane $I64X2 y 1 (zero_reg))))
+ (res_1 Reg (copy_reg $I64 (regpair_hi pair_1))))
+ (mov_to_vec128 $I64X2 res_0 res_1)))
+
;;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -282,6 +420,55 @@
(let ((pair RegPair (smul_wide x y)))
(copy_reg $I64 (regpair_hi pair))))
+;; Multiply high part signed, vector types with 8-, 16-, or 32-bit elements.
+(rule (lower (has_type $I8X16 (smulhi x y))) (vec_smulhi $I8X16 x y))
+(rule (lower (has_type $I16X8 (smulhi x y))) (vec_smulhi $I16X8 x y))
+(rule (lower (has_type $I32X4 (smulhi x y))) (vec_smulhi $I32X4 x y))
+
+;; Multiply high part unsigned, vector types with 64-bit elements.
+;; Has to be scalarized.
+(rule (lower (has_type $I64X2 (smulhi x y)))
+ (let ((pair_0 RegPair (smul_wide (vec_extract_lane $I64X2 x 0 (zero_reg))
+ (vec_extract_lane $I64X2 y 0 (zero_reg))))
+ (res_0 Reg (copy_reg $I64 (regpair_hi pair_0)))
+ (pair_1 RegPair (smul_wide (vec_extract_lane $I64X2 x 1 (zero_reg))
+ (vec_extract_lane $I64X2 y 1 (zero_reg))))
+ (res_1 Reg (copy_reg $I64 (regpair_hi pair_1))))
+ (mov_to_vec128 $I64X2 res_0 res_1)))
+
+
+;;;; Rules for `widening_pairwise_dot_product_s` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Widening pairwise dot product of two vector registers.
+(rule (lower (has_type dst_ty (widening_pairwise_dot_product_s
+ x @ (value_type src_ty) y)))
+ (vec_add dst_ty (vec_smul_even src_ty x y)
+ (vec_smul_odd src_ty x y)))
+
+
+;;;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Fixed-point multiplication of two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (sqmul_round_sat x y)))
+ (vec_pack_ssat (vec_widen_type ty)
+ (sqmul_impl (vec_widen_type ty)
+ (vec_unpacks_high ty x)
+ (vec_unpacks_high ty y))
+ (sqmul_impl (vec_widen_type ty)
+ (vec_unpacks_low ty x)
+ (vec_unpacks_low ty y))))
+
+;; Helper to perform the rounded multiply in the wider type.
+(decl sqmul_impl (Type Reg Reg) Reg)
+(rule (sqmul_impl $I32X4 x y)
+ (vec_ashr_imm $I32X4 (vec_add $I32X4 (vec_mul_impl $I32X4 x y)
+ (vec_imm_bit_mask $I32X4 17 17))
+ 15))
+(rule (sqmul_impl $I64X2 x y)
+ (vec_ashr_imm $I64X2 (vec_add $I64X2 (vec_mul_impl $I64X2 x y)
+ (vec_imm_bit_mask $I64X2 33 33))
+ 31))
+
;;;; Rules for `udiv` and `urem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -487,6 +674,15 @@
(let ((masked_amt u8 (mask_amt_imm ty y)))
(lshl_imm ty x masked_amt)))
+;; Vector shift left, shift amount in register.
+(rule (lower (has_type (ty_vec128 ty) (ishl x y)))
+ (vec_lshl_reg ty x y))
+
+;; Vector shift left, immediate shift amount.
+(rule (lower (has_type (ty_vec128 ty) (ishl x (i64_from_value y))))
+ (let ((masked_amt u8 (mask_amt_imm ty y)))
+ (vec_lshl_imm ty x masked_amt)))
+
;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -504,6 +700,15 @@
(masked_amt u8 (mask_amt_imm ty y)))
(lshr_imm (ty_ext32 ty) ext_reg masked_amt)))
+;; Vector shift right logical, shift amount in register.
+(rule (lower (has_type (ty_vec128 ty) (ushr x y)))
+ (vec_lshr_reg ty x y))
+
+;; Vector shift right logical, immediate shift amount.
+(rule (lower (has_type (ty_vec128 ty) (ushr x (i64_from_value y))))
+ (let ((masked_amt u8 (mask_amt_imm ty y)))
+ (vec_lshr_imm ty x masked_amt)))
+
;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -521,6 +726,15 @@
(masked_amt u8 (mask_amt_imm ty y)))
(ashr_imm (ty_ext32 ty) ext_reg masked_amt)))
+;; Vector shift right arithmetic, shift amount in register.
+(rule (lower (has_type (ty_vec128 ty) (sshr x y)))
+ (vec_ashr_reg ty x y))
+
+;; Vector shift right arithmetic, immediate shift amount.
+(rule (lower (has_type (ty_vec128 ty) (sshr x (i64_from_value y))))
+ (let ((masked_amt u8 (mask_amt_imm ty y)))
+ (vec_ashr_imm ty x masked_amt)))
+
;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -556,6 +770,15 @@
(or_reg ty (lshl_imm ext_ty ext_reg masked_pos_amt)
(lshr_imm ext_ty ext_reg masked_neg_amt))))
+;; Vector rotate left, shift amount in register.
+(rule (lower (has_type (ty_vec128 ty) (rotl x y)))
+ (vec_rot_reg ty x y))
+
+;; Vector rotate left, immediate shift amount.
+(rule (lower (has_type (ty_vec128 ty) (rotl x (i64_from_value y))))
+ (let ((masked_amt u8 (mask_amt_imm ty y)))
+ (vec_rot_imm ty x masked_amt)))
+
;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -594,6 +817,19 @@
(or_reg ty (lshl_imm ext_ty ext_reg masked_neg_amt)
(lshr_imm ext_ty ext_reg masked_pos_amt))))
+;; Vector rotate right, shift amount in register.
+;; Implemented as rotate left with negated rotate amount.
+(rule (lower (has_type (ty_vec128 ty) (rotr x y)))
+ (let ((negated_amt Reg (neg_reg $I32 y)))
+ (vec_rot_reg ty x negated_amt)))
+
+;; Vector rotate right, immediate shift amount.
+;; Implemented as rotate left with negated rotate amount.
+(rule (lower (has_type (ty_vec128 ty) (rotr x (i64_from_negated_value y))))
+ (let ((negated_amt u8 (mask_amt_imm ty y)))
+ (vec_rot_imm ty x negated_amt)))
+
+
;;;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Always a no-op.
@@ -623,6 +859,49 @@
(put_in_reg_sext64 x))
+;;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (snarrow x @ (value_type (ty_vec128 ty)) y))
+ (vec_pack_ssat ty y x))
+
+
+;;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (uunarrow x @ (value_type (ty_vec128 ty)) y))
+ (vec_pack_usat ty y x))
+
+
+;;;; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (unarrow x @ (value_type (ty_vec128 ty)) y))
+ (let ((zero Reg (vec_imm ty 0)))
+ (vec_pack_usat ty (vec_smax ty y zero) (vec_smax ty x zero))))
+
+
+;;;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (swiden_low x @ (value_type (ty_vec128 ty))))
+ (vec_unpacks_low ty x))
+
+
+;;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (swiden_high x @ (value_type (ty_vec128 ty))))
+ (vec_unpacks_high ty x))
+
+
+;;;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (uwiden_low x @ (value_type (ty_vec128 ty))))
+ (vec_unpacku_low ty x))
+
+
+;;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (uwiden_high x @ (value_type (ty_vec128 ty))))
+ (vec_unpacku_high ty x))
+
+
;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; z15 version using a single instruction (NOR).
@@ -634,6 +913,10 @@
(rule (lower (has_type (and (mie2_disabled) (fits_in_64 ty)) (bnot x)))
(not_reg ty x))
+;; Vector version using vector NOR.
+(rule (lower (has_type (ty_vec128 ty) (bnot x)))
+ (vec_not ty x))
+
;;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -657,6 +940,9 @@
(rule (lower (has_type (fits_in_64 ty) (band (sinkable_load_32_64 x) y)))
(and_mem ty y (sink_load x)))
+;; And two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (band x y)))
+ (vec_and ty x y))
;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -680,6 +966,10 @@
(rule (lower (has_type (fits_in_64 ty) (bor (sinkable_load_32_64 x) y)))
(or_mem ty y (sink_load x)))
+;; Or two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (bor x y)))
+ (vec_or ty x y))
+
;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -699,6 +989,10 @@
(rule (lower (has_type (fits_in_64 ty) (bxor (sinkable_load_32_64 x) y)))
(xor_mem ty y (sink_load x)))
+;; Xor two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (bxor x y)))
+ (vec_xor ty x y))
+
;;;; Rules for `band_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -710,6 +1004,10 @@
(rule (lower (has_type (and (mie2_disabled) (fits_in_64 ty)) (band_not x y)))
(and_reg ty x (not_reg ty y)))
+;; And-not two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (band_not x y)))
+ (vec_and_not ty x y))
+
;;;; Rules for `bor_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -721,6 +1019,10 @@
(rule (lower (has_type (and (mie2_disabled) (fits_in_64 ty)) (bor_not x y)))
(or_reg ty x (not_reg ty y)))
+;; Or-not two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (bor_not x y)))
+ (vec_or_not ty x y))
+
;;;; Rules for `bxor_not` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -732,6 +1034,10 @@
(rule (lower (has_type (and (mie2_disabled) (fits_in_64 ty)) (bxor_not x y)))
(not_reg ty (xor_reg ty x y)))
+;; Xor-not two vector registers.
+(rule (lower (has_type (ty_vec128 ty) (bxor_not x y)))
+ (vec_not_xor ty x y))
+
;;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -749,6 +1055,17 @@
(if_false Reg (and_reg ty z (not_reg ty rx))))
(or_reg ty if_false if_true)))
+;; Bitselect vector registers.
+(rule (lower (has_type (ty_vec128 ty) (bitselect x y z)))
+ (vec_select ty y z x))
+
+
+;;;; Rules for `vselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Vector select.
+(rule (lower (has_type (ty_vec128 ty) (vselect x y z)))
+ (vec_select ty y z x))
+
;;;; Rules for `breduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -917,6 +1234,10 @@
(cnt1 Reg (add_reg $I64 cnt2 (lshl_imm $I64 cnt2 8))))
(lshr_imm $I64 cnt1 56)))
+;; Population count for vector types.
+(rule (lower (has_type (ty_vec128 ty) (popcnt x)))
+ (vec_popcnt ty x))
+
;;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -960,6 +1281,20 @@
(fmax_reg ty x y))
+;;;; Rules for `fmin_pseudo` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Minimum of two registers.
+(rule (lower (has_type ty (fmin_pseudo x y)))
+ (fmin_pseudo_reg ty x y))
+
+
+;;;; Rules for `fmax_pseudo` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Maximum of two registers.
+(rule (lower (has_type ty (fmax_pseudo x y)))
+ (fmax_pseudo_reg ty x y))
+
+
;;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Copysign of two registers.
@@ -967,6 +1302,10 @@
(vec_select $F32 x y (imm $F32 2147483647)))
(rule (lower (has_type $F64 (fcopysign x y)))
(vec_select $F64 x y (imm $F64 9223372036854775807)))
+(rule (lower (has_type $F32X4 (fcopysign x y)))
+ (vec_select $F32X4 x y (vec_imm_bit_mask $F32X4 1 31)))
+(rule (lower (has_type $F64X2 (fcopysign x y)))
+ (vec_select $F64X2 x y (vec_imm_bit_mask $F64X2 1 63)))
;;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1028,35 +1367,73 @@
;;;; Rules for `fpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Promote a register.
-(rule (lower (has_type dst_ty (fpromote x @ (value_type src_ty))))
+(rule (lower (has_type (fits_in_64 dst_ty) (fpromote x @ (value_type src_ty))))
(fpromote_reg dst_ty src_ty x))
+;;;; Rules for `fvpromote_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Promote a register.
+(rule (lower (has_type $F64X2 (fvpromote_low x @ (value_type $F32X4))))
+ (fpromote_reg $F64X2 $F32X4 (vec_merge_low $I32X4 x x)))
+
+
;;;; Rules for `fdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Demote a register.
-(rule (lower (has_type dst_ty (fdemote x @ (value_type src_ty))))
+(rule (lower (has_type (fits_in_64 dst_ty) (fdemote x @ (value_type src_ty))))
(fdemote_reg dst_ty src_ty (FpuRoundMode.Current) x))
+;;;; Rules for `fvdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Demote a register.
+(rule (lower (has_type $F32X4 (fvdemote x @ (value_type $F64X2))))
+ (let ((dst Reg (fdemote_reg $F32X4 $F64X2 (FpuRoundMode.Current) x)))
+ (vec_permute $F32X4 dst (vec_imm $F32X4 0)
+ (vec_imm $I8X16 (imm8x16 16 16 16 16 16 16 16 16
+ 0 1 2 3 8 9 10 11)))))
+
+
;;;; Rules for `fcvt_from_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Convert a 32-bit or smaller unsigned integer to $F32 (z15 instruction).
(rule (lower (has_type $F32
(fcvt_from_uint x @ (value_type (and (vxrs_ext2_enabled) (fits_in_32 ty))))))
(fcvt_from_uint_reg $F32 (FpuRoundMode.ToNearestTiesToEven)
- (mov_to_fpr32 (put_in_reg_zext32 x))))
+ (put_in_reg_zext32 x)))
;; Convert a 64-bit or smaller unsigned integer to $F32, via an intermediate $F64.
(rule (lower (has_type $F32 (fcvt_from_uint x @ (value_type (fits_in_64 ty)))))
(fdemote_reg $F32 $F64 (FpuRoundMode.ToNearestTiesToEven)
(fcvt_from_uint_reg $F64 (FpuRoundMode.ShorterPrecision)
- (mov_to_fpr64 (put_in_reg_zext64 x)))))
+ (put_in_reg_zext64 x))))
;; Convert a 64-bit or smaller unsigned integer to $F64.
(rule (lower (has_type $F64 (fcvt_from_uint x @ (value_type (fits_in_64 ty)))))
(fcvt_from_uint_reg $F64 (FpuRoundMode.ToNearestTiesToEven)
- (mov_to_fpr64 (put_in_reg_zext64 x))))
+ (put_in_reg_zext64 x)))
+
+;; Convert $I32X4 to $F32X4 (z15 instruction).
+(rule (lower (has_type (and (vxrs_ext2_enabled) $F32X4)
+ (fcvt_from_uint x @ (value_type $I32X4))))
+ (fcvt_from_uint_reg $F32X4 (FpuRoundMode.ToNearestTiesToEven) x))
+
+;; Convert $I32X4 to $F32X4 (via two $F64X2 on z14).
+(rule (lower (has_type (and (vxrs_ext2_disabled) $F32X4)
+ (fcvt_from_uint x @ (value_type $I32X4))))
+ (vec_permute $F32X4
+ (fdemote_reg $F32X4 $F64X2 (FpuRoundMode.ToNearestTiesToEven)
+ (fcvt_from_uint_reg $F64X2 (FpuRoundMode.ShorterPrecision)
+ (vec_unpacku_high $I32X4 x)))
+ (fdemote_reg $F32X4 $F64X2 (FpuRoundMode.ToNearestTiesToEven)
+ (fcvt_from_uint_reg $F64X2 (FpuRoundMode.ShorterPrecision)
+ (vec_unpacku_low $I32X4 x)))
+ (vec_imm $I8X16 (imm8x16 0 1 2 3 8 9 10 11 16 17 18 19 24 25 26 27))))
+
+;; Convert $I64X2 to $F64X2.
+(rule (lower (has_type $F64X2 (fcvt_from_uint x @ (value_type $I64X2))))
+ (fcvt_from_uint_reg $F64X2 (FpuRoundMode.ToNearestTiesToEven) x))
;;;; Rules for `fcvt_from_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1065,25 +1442,55 @@
(rule (lower (has_type $F32
(fcvt_from_sint x @ (value_type (and (vxrs_ext2_enabled) (fits_in_32 ty))))))
(fcvt_from_sint_reg $F32 (FpuRoundMode.ToNearestTiesToEven)
- (mov_to_fpr32 (put_in_reg_sext32 x))))
+ (put_in_reg_sext32 x)))
;; Convert a 64-bit or smaller signed integer to $F32, via an intermediate $F64.
(rule (lower (has_type $F32 (fcvt_from_sint x @ (value_type (fits_in_64 ty)))))
(fdemote_reg $F32 $F64 (FpuRoundMode.ToNearestTiesToEven)
(fcvt_from_sint_reg $F64 (FpuRoundMode.ShorterPrecision)
- (mov_to_fpr64 (put_in_reg_sext64 x)))))
+ (put_in_reg_sext64 x))))
;; Convert a 64-bit or smaller signed integer to $F64.
(rule (lower (has_type $F64 (fcvt_from_sint x @ (value_type (fits_in_64 ty)))))
(fcvt_from_sint_reg $F64 (FpuRoundMode.ToNearestTiesToEven)
- (mov_to_fpr64 (put_in_reg_sext64 x))))
+ (put_in_reg_sext64 x)))
+
+;; Convert $I32X4 to $F32X4 (z15 instruction).
+(rule (lower (has_type (and (vxrs_ext2_enabled) $F32X4)
+ (fcvt_from_sint x @ (value_type $I32X4))))
+ (fcvt_from_sint_reg $F32X4 (FpuRoundMode.ToNearestTiesToEven) x))
+
+;; Convert $I32X4 to $F32X4 (via two $F64X2 on z14).
+(rule (lower (has_type (and (vxrs_ext2_disabled) $F32X4)
+ (fcvt_from_sint x @ (value_type $I32X4))))
+ (vec_permute $F32X4
+ (fdemote_reg $F32X4 $F64X2 (FpuRoundMode.ToNearestTiesToEven)
+ (fcvt_from_sint_reg $F64X2 (FpuRoundMode.ShorterPrecision)
+ (vec_unpacks_high $I32X4 x)))
+ (fdemote_reg $F32X4 $F64X2 (FpuRoundMode.ToNearestTiesToEven)
+ (fcvt_from_sint_reg $F64X2 (FpuRoundMode.ShorterPrecision)
+ (vec_unpacks_low $I32X4 x)))
+ (vec_imm $I8X16 (imm8x16 0 1 2 3 8 9 10 11 16 17 18 19 24 25 26 27))))
+
+;; Convert $I64X2 to $F64X2.
+(rule (lower (has_type $F64X2 (fcvt_from_sint x @ (value_type $I64X2))))
+ (fcvt_from_sint_reg $F64X2 (FpuRoundMode.ToNearestTiesToEven) x))
+
+
+;;;; Rules for `fcvt_low_from_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Convert the low half of a $I32X4 to a $F64X2.
+(rule (lower (has_type $F64X2 (fcvt_low_from_sint x @ (value_type $I32X4))))
+ (fcvt_from_sint_reg $F64X2 (FpuRoundMode.ToNearestTiesToEven)
+ (vec_unpacks_low $I32X4 x)))
;;;; Rules for `fcvt_to_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Convert a floating-point value in a register to an unsigned integer value.
+;; Convert a scalar floating-point value in a register to an unsigned integer.
;; Traps if the input cannot be represented in the output type.
-(rule (lower (has_type dst_ty (fcvt_to_uint x @ (value_type src_ty))))
+(rule (lower (has_type (fits_in_64 dst_ty)
+ (fcvt_to_uint x @ (value_type src_ty))))
(let ((src Reg (put_in_reg x))
;; First, check whether the input is a NaN, and trap if so.
(_1 Reg (trap_if (fcmp_reg src_ty src src)
@@ -1104,9 +1511,10 @@
;;;; Rules for `fcvt_to_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Convert a floating-point value in a register to a signed integer value.
+;; Convert a scalar floating-point value in a register to a signed integer.
;; Traps if the input cannot be represented in the output type.
-(rule (lower (has_type dst_ty (fcvt_to_sint x @ (value_type src_ty))))
+(rule (lower (has_type (fits_in_64 dst_ty)
+ (fcvt_to_sint x @ (value_type src_ty))))
(let ((src Reg (put_in_reg x))
;; First, check whether the input is a NaN, and trap if so.
(_1 Reg (trap_if (fcmp_reg src_ty src src)
@@ -1128,8 +1536,9 @@
;;;; Rules for `fcvt_to_uint_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Convert a floating-point value in a register to an unsigned integer value.
-(rule (lower (has_type dst_ty (fcvt_to_uint_sat x @ (value_type src_ty))))
+;; Convert a scalar floating-point value in a register to an unsigned integer.
+(rule (lower (has_type (fits_in_64 dst_ty)
+ (fcvt_to_uint_sat x @ (value_type src_ty))))
(let ((src Reg (put_in_reg x))
;; Perform the conversion using the larger type size.
(flt_ty Type (fcvt_flt_ty dst_ty src_ty))
@@ -1139,11 +1548,30 @@
;; Clamp the output to the destination type bounds.
(uint_sat_reg dst_ty int_ty dst)))
+;; Convert $F32X4 to $I32X4 (z15 instruction).
+(rule (lower (has_type (and (vxrs_ext2_enabled) $I32X4)
+ (fcvt_to_uint_sat x @ (value_type $F32X4))))
+ (fcvt_to_uint_reg $F32X4 (FpuRoundMode.ToZero) x))
+
+;; Convert $F32X4 to $I32X4 (via two $F64X2 on z14).
+(rule (lower (has_type (and (vxrs_ext2_disabled) $I32X4)
+ (fcvt_to_uint_sat x @ (value_type $F32X4))))
+ (vec_pack_usat $I64X2
+ (fcvt_to_uint_reg $F64X2 (FpuRoundMode.ToZero)
+ (fpromote_reg $F64X2 $F32X4 (vec_merge_high $I32X4 x x)))
+ (fcvt_to_uint_reg $F64X2 (FpuRoundMode.ToZero)
+ (fpromote_reg $F64X2 $F32X4 (vec_merge_low $I32X4 x x)))))
+
+;; Convert $F64X2 to $I64X2.
+(rule (lower (has_type $I64X2 (fcvt_to_uint_sat x @ (value_type $F64X2))))
+ (fcvt_to_uint_reg $F64X2 (FpuRoundMode.ToZero) x))
+
;;;; Rules for `fcvt_to_sint_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Convert a floating-point value in a register to a signed integer value.
-(rule (lower (has_type dst_ty (fcvt_to_sint_sat x @ (value_type src_ty))))
+;; Convert a scalar floating-point value in a register to a signed integer.
+(rule (lower (has_type (fits_in_64 dst_ty)
+ (fcvt_to_sint_sat x @ (value_type src_ty))))
(let ((src Reg (put_in_reg x))
;; Perform the conversion using the larger type size.
(flt_ty Type (fcvt_flt_ty dst_ty src_ty))
@@ -1160,24 +1588,578 @@
;; Clamp the output to the destination type bounds.
(sint_sat_reg dst_ty int_ty sat)))
+;; Convert $F32X4 to $I32X4 (z15 instruction).
+(rule (lower (has_type (and (vxrs_ext2_enabled) $I32X4)
+ (fcvt_to_sint_sat src @ (value_type $F32X4))))
+ ;; See above for why we need to handle NaNs specially.
+ (vec_select $I32X4
+ (fcvt_to_sint_reg $F32X4 (FpuRoundMode.ToZero) src)
+ (vec_imm $I32X4 0) (vec_fcmpeq $F32X4 src src)))
+
+;; Convert $F32X4 to $I32X4 (via two $F64X2 on z14).
+(rule (lower (has_type (and (vxrs_ext2_disabled) $I32X4)
+ (fcvt_to_sint_sat src @ (value_type $F32X4))))
+ ;; See above for why we need to handle NaNs specially.
+ (vec_select $I32X4
+ (vec_pack_ssat $I64X2
+ (fcvt_to_sint_reg $F64X2 (FpuRoundMode.ToZero)
+ (fpromote_reg $F64X2 $F32X4 (vec_merge_high $I32X4 src src)))
+ (fcvt_to_sint_reg $F64X2 (FpuRoundMode.ToZero)
+ (fpromote_reg $F64X2 $F32X4 (vec_merge_low $I32X4 src src))))
+ (vec_imm $I32X4 0) (vec_fcmpeq $F32X4 src src)))
+
+;; Convert $F64X2 to $I64X2.
+(rule (lower (has_type $I64X2 (fcvt_to_sint_sat src @ (value_type $F64X2))))
+ ;; See above for why we need to handle NaNs specially.
+ (vec_select $I64X2
+ (fcvt_to_sint_reg $F64X2 (FpuRoundMode.ToZero) src)
+ (vec_imm $I64X2 0) (vec_fcmpeq $F64X2 src src)))
+
;;;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Reinterpret a 64-bit integer value as floating-point.
(rule (lower (has_type $F64 (bitcast x @ (value_type $I64))))
- (mov_to_fpr64 x))
+ (vec_insert_lane_undef $F64X2 x 0 (zero_reg)))
;; Reinterpret a 64-bit floating-point value as integer.
(rule (lower (has_type $I64 (bitcast x @ (value_type $F64))))
- (mov_from_fpr64 x))
+ (vec_extract_lane $F64X2 x 0 (zero_reg)))
-;; Reinterpret a 32-bit integer value as floating-point (via $I64).
+;; Reinterpret a 32-bit integer value as floating-point.
(rule (lower (has_type $F32 (bitcast x @ (value_type $I32))))
- (mov_to_fpr32 x))
+ (vec_insert_lane_undef $F32X4 x 0 (zero_reg)))
-;; Reinterpret a 32-bit floating-point value as integer (via $I64).
+;; Reinterpret a 32-bit floating-point value as integer.
(rule (lower (has_type $I32 (bitcast x @ (value_type $F32))))
- (mov_from_fpr32 x))
+ (vec_extract_lane $F32X4 x 0 (zero_reg)))
+
+
+;;;; Rules for `raw_bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Raw bitcast is always a no-op.
+(rule (lower (raw_bitcast x)) x)
+
+
+;;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Insert vector lane from general-purpose register.
+(rule (lower (insertlane x @ (value_type ty)
+ y @ (value_type (ty_int_bool_ref_scalar_64 _))
+ (u8_from_uimm8 idx)))
+ (vec_insert_lane ty x y (be_lane_idx ty idx) (zero_reg)))
+
+;; Insert vector lane from floating-point register.
+(rule (lower (insertlane x @ (value_type ty)
+ y @ (value_type (ty_scalar_float _))
+ (u8_from_uimm8 idx)))
+ (vec_move_lane_and_insert ty x (be_lane_idx ty idx) y 0))
+
+;; Insert vector lane from another vector lane.
+(rule (lower (insertlane x @ (value_type ty)
+ (extractlane y (u8_from_uimm8 src_idx))
+ (u8_from_uimm8 dst_idx)))
+ (vec_move_lane_and_insert ty x (be_lane_idx ty dst_idx)
+ y (be_lane_idx ty src_idx)))
+
+;; Insert vector lane from signed 16-bit immediate.
+(rule (lower (insertlane x @ (value_type ty) (i16_from_value y)
+ (u8_from_uimm8 idx)))
+ (vec_insert_lane_imm ty x y (be_lane_idx ty idx)))
+
+;; Insert vector lane from big-endian memory.
+(rule (lower (insertlane x @ (value_type ty) (sinkable_load y)
+ (u8_from_uimm8 idx)))
+ (vec_load_lane ty x (sink_load y) (be_lane_idx ty idx)))
+
+;; Insert vector lane from little-endian memory.
+(rule (lower (insertlane x @ (value_type ty) (sinkable_load_little y)
+ (u8_from_uimm8 idx)))
+ (vec_load_lane_little ty x (sink_load y) (be_lane_idx ty idx)))
+
+
+;; Helper to extract one lane from a vector and insert it into another.
+(decl vec_move_lane_and_insert (Type Reg u8 Reg u8) Reg)
+
+;; For 64-bit elements we always use VPDI.
+(rule (vec_move_lane_and_insert ty @ (multi_lane 64 _) dst 0 src src_idx)
+ (vec_permute_dw_imm ty src src_idx dst 1))
+(rule (vec_move_lane_and_insert ty @ (multi_lane 64 _) dst 1 src src_idx)
+ (vec_permute_dw_imm ty dst 0 src src_idx))
+
+;; If source and destination index are the same, use vec_select.
+(rule (vec_move_lane_and_insert ty dst idx src idx)
+ (vec_select ty src
+ dst (vec_imm_byte_mask ty (lane_byte_mask ty idx))))
+
+;; Otherwise replicate source first and then use vec_select.
+(rule (vec_move_lane_and_insert ty dst dst_idx src src_idx)
+ (vec_select ty (vec_replicate_lane ty src src_idx)
+ dst (vec_imm_byte_mask ty (lane_byte_mask ty dst_idx))))
+
+
+;; Helper to implement a generic little-endian variant of vec_load_lane.
+(decl vec_load_lane_little (Type Reg MemArg u8) Reg)
+
+;; 8-byte little-endian loads can be performed via a normal load.
+(rule (vec_load_lane_little ty @ (multi_lane 8 _) dst addr lane_imm)
+ (vec_load_lane ty dst addr lane_imm))
+
+;; On z15, we have instructions to perform little-endian loads.
+(rule (vec_load_lane_little (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 16 _)) dst addr lane_imm)
+ (vec_load_lane_rev ty dst addr lane_imm))
+(rule (vec_load_lane_little (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 32 _)) dst addr lane_imm)
+ (vec_load_lane_rev ty dst addr lane_imm))
+(rule (vec_load_lane_little (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 64 _)) dst addr lane_imm)
+ (vec_load_lane_rev ty dst addr lane_imm))
+
+;; On z14, use a little-endian load to GPR followed by vec_insert_lane.
+(rule (vec_load_lane_little (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 16 _)) dst addr lane_imm)
+ (vec_insert_lane ty dst (loadrev16 addr) lane_imm (zero_reg)))
+(rule (vec_load_lane_little (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 32 _)) dst addr lane_imm)
+ (vec_insert_lane ty dst (loadrev32 addr) lane_imm (zero_reg)))
+(rule (vec_load_lane_little (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 64 _)) dst addr lane_imm)
+ (vec_insert_lane ty dst (loadrev64 addr) lane_imm (zero_reg)))
+
+;; Helper to implement a generic little-endian variant of vec_load_lane_undef.
+(decl vec_load_lane_little_undef (Type MemArg u8) Reg)
+
+;; 8-byte little-endian loads can be performed via a normal load.
+(rule (vec_load_lane_little_undef ty @ (multi_lane 8 _) addr lane_imm)
+ (vec_load_lane_undef ty addr lane_imm))
+
+;; On z15, we have instructions to perform little-endian loads.
+(rule (vec_load_lane_little_undef (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 16 _)) addr lane_imm)
+ (vec_load_lane_rev_undef ty addr lane_imm))
+(rule (vec_load_lane_little_undef (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 32 _)) addr lane_imm)
+ (vec_load_lane_rev_undef ty addr lane_imm))
+(rule (vec_load_lane_little_undef (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 64 _)) addr lane_imm)
+ (vec_load_lane_rev_undef ty addr lane_imm))
+
+;; On z14, use a little-endian load to GPR followed by vec_insert_lane_undef.
+(rule (vec_load_lane_little_undef (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 16 _)) addr lane_imm)
+ (vec_insert_lane_undef ty (loadrev16 addr) lane_imm (zero_reg)))
+(rule (vec_load_lane_little_undef (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 32 _)) addr lane_imm)
+ (vec_insert_lane_undef ty (loadrev32 addr) lane_imm (zero_reg)))
+(rule (vec_load_lane_little_undef (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 64 _)) addr lane_imm)
+ (vec_insert_lane_undef ty (loadrev64 addr) lane_imm (zero_reg)))
+
+
+;;;; Rules for `extractlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Extract vector lane to general-purpose register.
+(rule (lower (has_type (ty_int_bool_ref_scalar_64 _)
+ (extractlane x @ (value_type ty) (u8_from_uimm8 idx))))
+ (vec_extract_lane ty x (be_lane_idx ty idx) (zero_reg)))
+
+;; Extract vector lane to floating-point register.
+(rule (lower (has_type (ty_scalar_float _)
+ (extractlane x @ (value_type ty) (u8_from_uimm8 idx))))
+ (vec_replicate_lane ty x (be_lane_idx ty idx)))
+
+;; Extract vector lane and store to big-endian memory.
+(rule (lower (store flags @ (bigendian)
+ (extractlane x @ (value_type ty) (u8_from_uimm8 idx))
+ addr offset))
+ (side_effect (vec_store_lane ty x
+ (lower_address flags addr offset) (be_lane_idx ty idx))))
+
+;; Extract vector lane and store to little-endian memory.
+(rule (lower (store flags @ (littleendian)
+ (extractlane x @ (value_type ty) (u8_from_uimm8 idx))
+ addr offset))
+ (side_effect (vec_store_lane_little ty x
+ (lower_address flags addr offset) (be_lane_idx ty idx))))
+
+
+;; Helper to implement a generic little-endian variant of vec_store_lane.
+(decl vec_store_lane_little (Type Reg MemArg u8) SideEffectNoResult)
+
+;; 8-byte little-endian stores can be performed via a normal store.
+(rule (vec_store_lane_little ty @ (multi_lane 8 _) src addr lane_imm)
+ (vec_store_lane ty src addr lane_imm))
+
+;; On z15, we have instructions to perform little-endian stores.
+(rule (vec_store_lane_little (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 16 _)) src addr lane_imm)
+ (vec_store_lane_rev ty src addr lane_imm))
+(rule (vec_store_lane_little (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 32 _)) src addr lane_imm)
+ (vec_store_lane_rev ty src addr lane_imm))
+(rule (vec_store_lane_little (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 64 _)) src addr lane_imm)
+ (vec_store_lane_rev ty src addr lane_imm))
+
+;; On z14, use vec_extract_lane followed by a little-endian store from GPR.
+(rule (vec_store_lane_little (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 16 _)) src addr lane_imm)
+ (storerev16 (vec_extract_lane ty src lane_imm (zero_reg)) addr))
+(rule (vec_store_lane_little (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 32 _)) src addr lane_imm)
+ (storerev32 (vec_extract_lane ty src lane_imm (zero_reg)) addr))
+(rule (vec_store_lane_little (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 64 _)) src addr lane_imm)
+ (storerev64 (vec_extract_lane ty src lane_imm (zero_reg)) addr))
+
+
+;;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Load replicated value from general-purpose register.
+(rule (lower (has_type ty (splat
+ x @ (value_type (ty_int_bool_ref_scalar_64 _)))))
+ (vec_replicate_lane ty (vec_insert_lane_undef ty x 0 (zero_reg)) 0))
+
+;; Load replicated value from floating-point register.
+(rule (lower (has_type ty (splat
+ x @ (value_type (ty_scalar_float _)))))
+ (vec_replicate_lane ty x 0))
+
+;; Load replicated value from vector lane.
+(rule (lower (has_type ty (splat (extractlane x (u8_from_uimm8 idx)))))
+ (vec_replicate_lane ty x (be_lane_idx ty idx)))
+
+;; Load replicated 16-bit immediate value.
+(rule (lower (has_type ty (splat (i16_from_value x))))
+ (vec_imm_replicate ty x))
+
+;; Load replicated value from big-endian memory.
+(rule (lower (has_type ty (splat (sinkable_load x))))
+ (vec_load_replicate ty (sink_load x)))
+
+;; Load replicated value from little-endian memory.
+(rule (lower (has_type ty (splat (sinkable_load_little x))))
+ (vec_load_replicate_little ty (sink_load x)))
+
+
+;; Helper to implement a generic little-endian variant of vec_load_replicate
+(decl vec_load_replicate_little (Type MemArg) Reg)
+
+;; 8-byte little-endian loads can be performed via a normal load.
+(rule (vec_load_replicate_little ty @ (multi_lane 8 _) addr)
+ (vec_load_replicate ty addr))
+
+;; On z15, we have instructions to perform little-endian loads.
+(rule (vec_load_replicate_little (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 16 _)) addr)
+ (vec_load_replicate_rev ty addr))
+(rule (vec_load_replicate_little (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 32 _)) addr)
+ (vec_load_replicate_rev ty addr))
+(rule (vec_load_replicate_little (and (vxrs_ext2_enabled)
+ ty @ (multi_lane 64 _)) addr)
+ (vec_load_replicate_rev ty addr))
+
+;; On z14, use a little-endian load (via GPR) and replicate.
+(rule (vec_load_replicate_little (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 16 _)) addr)
+ (vec_replicate_lane ty (vec_load_lane_little_undef ty addr 0) 0))
+(rule (vec_load_replicate_little (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 32 _)) addr)
+ (vec_replicate_lane ty (vec_load_lane_little_undef ty addr 0) 0))
+(rule (vec_load_replicate_little (and (vxrs_ext2_disabled)
+ ty @ (multi_lane 64 _)) addr)
+ (vec_replicate_lane ty (vec_load_lane_little_undef ty addr 0) 0))
+
+
+;;;; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Load scalar value from general-purpose register.
+(rule (lower (has_type ty (scalar_to_vector
+ x @ (value_type (ty_int_bool_ref_scalar_64 _)))))
+ (vec_insert_lane ty (vec_imm ty 0) x (be_lane_idx ty 0) (zero_reg)))
+
+;; Load scalar value from floating-point register.
+(rule (lower (has_type ty (scalar_to_vector
+ x @ (value_type (ty_scalar_float _)))))
+ (vec_move_lane_and_zero ty (be_lane_idx ty 0) x 0))
+
+;; Load scalar value from vector lane.
+(rule (lower (has_type ty (scalar_to_vector
+ (extractlane x (u8_from_uimm8 idx)))))
+ (vec_move_lane_and_zero ty (be_lane_idx ty 0) x (be_lane_idx ty idx)))
+
+;; Load scalar 16-bit immediate value.
+(rule (lower (has_type ty (scalar_to_vector (i16_from_value x))))
+ (vec_insert_lane_imm ty (vec_imm ty 0) x (be_lane_idx ty 0)))
+
+;; Load scalar value from big-endian memory.
+(rule (lower (has_type ty (scalar_to_vector (sinkable_load x))))
+ (vec_load_lane ty (vec_imm ty 0) (sink_load x) (be_lane_idx ty 0)))
+
+;; Load scalar value lane from little-endian memory.
+(rule (lower (has_type ty (scalar_to_vector (sinkable_load_little x))))
+ (vec_load_lane_little ty (vec_imm ty 0) (sink_load x) (be_lane_idx ty 0)))
+
+
+;; Helper to extract one lane from a vector and insert it into a zero vector.
+(decl vec_move_lane_and_zero (Type u8 Reg u8) Reg)
+
+;; For 64-bit elements we always use VPDI.
+(rule (vec_move_lane_and_zero ty @ (multi_lane 64 _) 0 src src_idx)
+ (vec_permute_dw_imm ty src src_idx (vec_imm ty 0) 0))
+(rule (vec_move_lane_and_zero ty @ (multi_lane 64 _) 1 src src_idx)
+ (vec_permute_dw_imm ty (vec_imm ty 0) 0 src src_idx))
+
+;; If source and destination index are the same, simply mask to this lane.
+(rule (vec_move_lane_and_zero ty idx src idx)
+ (vec_and ty src
+ (vec_imm_byte_mask ty (lane_byte_mask ty idx))))
+
+;; Otherwise replicate source first and then mask to the lane.
+(rule (vec_move_lane_and_zero ty dst_idx src src_idx)
+ (vec_and ty (vec_replicate_lane ty src src_idx)
+ (vec_imm_byte_mask ty (lane_byte_mask ty dst_idx))))
+
+
+;;;; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; General case: use vec_permute and then mask off zero lanes.
+(rule (lower (shuffle x y (shuffle_mask permute_mask and_mask)))
+ (vec_and $I8X16 (vec_imm_byte_mask $I8X16 and_mask)
+ (vec_permute $I8X16 x y (vec_imm $I8X16 permute_mask))))
+
+;; If the pattern has no zero lanes, just a vec_permute suffices.
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (vec_permute $I8X16 x y (vec_imm $I8X16 permute_mask)))
+
+;; Special patterns that can be implemented via MERGE HIGH.
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23) permute_mask)
+ (vec_merge_high $I64X2 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 1 2 3 16 17 18 19 4 5 6 7 20 21 22 23) permute_mask)
+ (vec_merge_high $I32X4 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 1 16 17 2 3 18 19 4 5 20 21 6 7 22 23) permute_mask)
+ (vec_merge_high $I16X8 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 16 1 17 2 18 3 19 4 20 5 21 6 22 7 23) permute_mask)
+ (vec_merge_high $I8X16 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 17 18 19 20 21 22 23 0 1 2 3 4 5 6 7) permute_mask)
+ (vec_merge_high $I64X2 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 17 18 19 0 1 2 3 20 21 22 23 4 5 6 7) permute_mask)
+ (vec_merge_high $I32X4 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 17 0 1 18 19 2 3 20 21 4 5 22 23 6 7) permute_mask)
+ (vec_merge_high $I16X8 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 0 17 1 18 2 19 3 20 4 21 5 22 6 23 7) permute_mask)
+ (vec_merge_high $I8X16 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7) permute_mask)
+ (vec_merge_high $I64X2 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 1 2 3 0 1 2 3 4 5 6 7 4 5 6 7) permute_mask)
+ (vec_merge_high $I32X4 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 1 0 1 2 3 2 3 4 5 4 5 6 7 6 7) permute_mask)
+ (vec_merge_high $I16X8 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7) permute_mask)
+ (vec_merge_high $I8X16 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 17 18 19 20 21 22 23 16 17 18 19 20 21 22 23) permute_mask)
+ (vec_merge_high $I64X2 y y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 17 18 19 16 17 18 19 20 21 22 23 20 21 22 23) permute_mask)
+ (vec_merge_high $I32X4 y y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 17 16 17 18 19 18 19 20 21 20 21 22 23 22 23) permute_mask)
+ (vec_merge_high $I16X8 y y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 16 17 17 18 18 19 19 20 20 21 21 22 22 23 23) permute_mask)
+ (vec_merge_high $I8X16 y y))
+
+;; Special patterns that can be implemented via MERGE LOW.
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31) permute_mask)
+ (vec_merge_low $I64X2 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 9 10 11 24 25 26 27 12 13 14 15 28 29 30 31) permute_mask)
+ (vec_merge_low $I32X4 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 9 24 25 10 11 26 27 12 13 28 29 14 15 30 31) permute_mask)
+ (vec_merge_low $I16X8 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 24 9 25 10 26 11 27 12 28 13 29 14 30 15 31) permute_mask)
+ (vec_merge_low $I8X16 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 25 26 27 28 29 30 31 8 9 10 11 12 13 14 15) permute_mask)
+ (vec_merge_low $I64X2 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 25 26 27 8 9 10 11 28 29 30 31 12 13 14 15) permute_mask)
+ (vec_merge_low $I32X4 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 25 8 9 26 27 10 11 28 29 12 13 30 31 14 15) permute_mask)
+ (vec_merge_low $I16X8 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 8 25 9 26 10 27 11 28 12 29 13 30 14 31 15) permute_mask)
+ (vec_merge_low $I8X16 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15) permute_mask)
+ (vec_merge_low $I64X2 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 9 10 11 8 9 10 11 12 13 14 15 12 13 14 15) permute_mask)
+ (vec_merge_low $I32X4 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 9 8 9 10 11 10 11 12 13 12 13 14 15 14 15) permute_mask)
+ (vec_merge_low $I16X8 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 8 9 9 10 10 11 11 12 12 13 13 14 14 15 15) permute_mask)
+ (vec_merge_low $I8X16 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 25 26 27 28 29 30 31 24 25 26 27 28 29 30 31) permute_mask)
+ (vec_merge_low $I64X2 y y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 25 26 27 24 25 26 27 28 29 30 31 28 29 30 31) permute_mask)
+ (vec_merge_low $I32X4 y y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 25 24 25 26 27 26 27 28 29 28 29 30 31 30 31) permute_mask)
+ (vec_merge_low $I16X8 y y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 24 25 25 26 26 27 27 28 28 29 29 30 30 31 31) permute_mask)
+ (vec_merge_low $I8X16 y y))
+
+;; Special patterns that can be implemented via PACK.
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 4 5 6 7 12 13 14 15 20 21 22 23 28 29 30 31) permute_mask)
+ (vec_pack $I64X2 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 2 3 6 7 10 11 14 15 18 19 22 23 26 27 30 31) permute_mask)
+ (vec_pack $I32X4 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31) permute_mask)
+ (vec_pack $I16X8 x y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 20 21 22 23 28 29 30 31 4 5 6 7 12 13 14 15) permute_mask)
+ (vec_pack $I64X2 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 18 19 22 23 26 27 30 31 2 3 6 7 10 11 14 15) permute_mask)
+ (vec_pack $I32X4 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 17 19 21 23 25 27 29 31 1 3 5 7 9 11 13 15) permute_mask)
+ (vec_pack $I16X8 y x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 4 5 6 7 12 13 14 15 4 5 6 7 12 13 14 15) permute_mask)
+ (vec_pack $I64X2 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 2 3 6 7 10 11 14 15 2 3 6 7 10 11 14 15) permute_mask)
+ (vec_pack $I32X4 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 1 3 5 7 9 11 13 15 1 3 5 7 9 11 13 15) permute_mask)
+ (vec_pack $I16X8 x x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 20 21 22 23 28 29 30 31 20 21 22 23 28 29 30 31) permute_mask)
+ (vec_pack $I64X2 y y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 18 19 22 23 26 27 30 31 18 19 22 23 26 27 30 31) permute_mask)
+ (vec_pack $I32X4 y y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 17 19 21 23 25 27 29 31 17 19 21 23 25 27 29 31) permute_mask)
+ (vec_pack $I16X8 y y))
+
+;; Special patterns that can be implemented via UNPACK HIGH.
+(rule (lower (shuffle x y (shuffle_mask permute_mask 3855)))
+ (if-let (imm8x16 _ _ _ _ 0 1 2 3 _ _ _ _ 4 5 6 7) permute_mask)
+ (vec_unpacku_high $I32X4 x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 13107)))
+ (if-let (imm8x16 _ _ 0 1 _ _ 2 3 _ _ 4 5 _ _ 6 7) permute_mask)
+ (vec_unpacku_high $I16X8 x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 21845)))
+ (if-let (imm8x16 _ 0 _ 1 _ 2 _ 3 _ 4 _ 5 _ 6 _ 7) permute_mask)
+ (vec_unpacku_high $I8X16 x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 3855)))
+ (if-let (imm8x16 _ _ _ _ 16 17 18 19 _ _ _ _ 20 21 22 23) permute_mask)
+ (vec_unpacku_high $I32X4 y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 13107)))
+ (if-let (imm8x16 _ _ 16 17 _ _ 18 19 _ _ 20 21 _ _ 22 23) permute_mask)
+ (vec_unpacku_high $I16X8 y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 21845)))
+ (if-let (imm8x16 _ 16 _ 17 _ 18 _ 19 _ 20 _ 21 _ 22 _ 23) permute_mask)
+ (vec_unpacku_high $I8X16 y))
+
+;; Special patterns that can be implemented via UNPACK LOW.
+(rule (lower (shuffle x y (shuffle_mask permute_mask 3855)))
+ (if-let (imm8x16 _ _ _ _ 8 9 10 11 _ _ _ _ 12 13 14 15) permute_mask)
+ (vec_unpacku_low $I32X4 x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 13107)))
+ (if-let (imm8x16 _ _ 8 9 _ _ 10 11 _ _ 12 13 _ _ 14 15) permute_mask)
+ (vec_unpacku_low $I16X8 x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 21845)))
+ (if-let (imm8x16 _ 8 _ 9 _ 10 _ 11 _ 12 _ 13 _ 14 _ 15) permute_mask)
+ (vec_unpacku_low $I8X16 x))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 3855)))
+ (if-let (imm8x16 _ _ _ _ 24 25 26 27 _ _ _ _ 28 29 30 31) permute_mask)
+ (vec_unpacku_low $I32X4 y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 13107)))
+ (if-let (imm8x16 _ _ 24 25 _ _ 26 27 _ _ 28 29 _ _ 30 31) permute_mask)
+ (vec_unpacku_low $I16X8 y))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 21845)))
+ (if-let (imm8x16 _ 24 _ 25 _ 26 _ 27 _ 28 _ 29 _ 30 _ 31) permute_mask)
+ (vec_unpacku_low $I8X16 y))
+
+;; Special patterns that can be implemented via PERMUTE DOUBLEWORD IMMEDIATE.
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 1 2 3 4 5 6 7 24 25 26 27 28 29 30 31) permute_mask)
+ (vec_permute_dw_imm $I8X16 x 0 y 1))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) permute_mask)
+ (vec_permute_dw_imm $I8X16 x 1 y 0))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 17 18 19 20 21 22 23 8 9 10 11 12 13 14 15) permute_mask)
+ (vec_permute_dw_imm $I8X16 y 0 x 1))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 25 26 27 28 29 30 31 0 1 2 3 4 5 6 7) permute_mask)
+ (vec_permute_dw_imm $I8X16 y 1 x 0))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15) permute_mask)
+ (vec_permute_dw_imm $I8X16 x 0 x 1))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 8 9 10 11 12 13 14 15 0 1 2 3 4 5 6 7) permute_mask)
+ (vec_permute_dw_imm $I8X16 x 1 x 0))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31) permute_mask)
+ (vec_permute_dw_imm $I8X16 y 0 y 1))
+(rule (lower (shuffle x y (shuffle_mask permute_mask 65535)))
+ (if-let (imm8x16 24 25 26 27 28 29 30 31 16 17 18 19 20 21 22 23) permute_mask)
+ (vec_permute_dw_imm $I8X16 y 1 y 0))
+
+
+;;;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; We need to modify the lane mask at runtime in two ways:
+;; - convert from little-endian to big-endian lane numbering
+;; - handle mask elements outside the range 0..15 by zeroing the lane
+;;
+;; To do so efficiently, we compute:
+;; permute-lane-element := umax (239, ~ swizzle-lane-element)
+;; which has the following effect:
+;; elements 0 .. 15 --> 255 .. 240 (i.e. 31 .. 16 mod 32)
+;; everything else --> 239 (i.e. 15 mod 32)
+;;
+;; Then, we can use a single permute instruction with
+;; a zero vector as first operand (covering lane 15)
+;; the input vector as second operand (covering lanes 16 .. 31)
+;; to implement the required swizzle semantics.
+
+(rule (lower (has_type (ty_vec128 ty) (swizzle x y)))
+ (vec_permute ty (vec_imm ty 0) x
+ (vec_umax $I8X16 (vec_imm_splat $I8X16 239)
+ (vec_not $I8X16 y))))
;;;; Rules for `stack_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1249,35 +2231,39 @@
(rule (lower (has_type $R64 (load flags @ (littleendian) addr offset)))
(loadrev64 (lower_address flags addr offset)))
-;; Load 32-bit big-endian floating-point values.
+;; Load 32-bit big-endian floating-point values (as vector lane).
(rule (lower (has_type $F32 (load flags @ (bigendian) addr offset)))
- (fpu_load32 (lower_address flags addr offset)))
+ (vec_load_lane_undef $F32X4 (lower_address flags addr offset) 0))
-;; Load 32-bit little-endian floating-point values (z15 instruction).
-(rule (lower (has_type (and (vxrs_ext2_enabled) $F32)
- (load flags @ (littleendian) addr offset)))
- (fpu_loadrev32 (lower_address flags addr offset)))
+;; Load 32-bit little-endian floating-point values (as vector lane).
+(rule (lower (has_type $F32 (load flags @ (littleendian) addr offset)))
+ (vec_load_lane_little_undef $F32X4 (lower_address flags addr offset) 0))
-;; Load 32-bit little-endian floating-point values (via GPR on z14).
-(rule (lower (has_type (and (vxrs_ext2_disabled) $F32)
- (load flags @ (littleendian) addr offset)))
- (let ((gpr Reg (loadrev32 (lower_address flags addr offset))))
- (mov_to_fpr32 gpr)))
-
-;; Load 64-bit big-endian floating-point values.
+;; Load 64-bit big-endian floating-point values (as vector lane).
(rule (lower (has_type $F64 (load flags @ (bigendian) addr offset)))
- (fpu_load64 (lower_address flags addr offset)))
+ (vec_load_lane_undef $F64X2 (lower_address flags addr offset) 0))
-;; Load 64-bit little-endian floating-point values (z15 instruction).
-(rule (lower (has_type (and (vxrs_ext2_enabled) $F64)
- (load flags @ (littleendian) addr offset)))
- (fpu_loadrev64 (lower_address flags addr offset)))
+;; Load 64-bit little-endian floating-point values (as vector lane).
+(rule (lower (has_type $F64 (load flags @ (littleendian) addr offset)))
+ (vec_load_lane_little_undef $F64X2 (lower_address flags addr offset) 0))
-;; Load 64-bit little-endian floating-point values (via GPR on z14).
-(rule (lower (has_type (and (vxrs_ext2_disabled) $F64)
- (load flags @ (littleendian) addr offset)))
- (let ((gpr Reg (loadrev64 (lower_address flags addr offset))))
- (mov_to_fpr64 gpr)))
+;; Load 128-bit big-endian vector values.
+(rule (lower (has_type (ty_vec128 ty) (load flags @ (bigendian) addr offset)))
+ (vec_load ty (lower_address flags addr offset)))
+
+;; Load 128-bit little-endian vector values (z15 instruction).
+(rule (lower (has_type (and (vxrs_ext2_enabled) (ty_vec128 ty))
+ (load flags @ (littleendian) addr offset)))
+ (vec_loadrev ty (lower_address flags addr offset)))
+
+;; Load 128-bit little-endian vector values (via GPRs on z14).
+(rule (lower (has_type (and (vxrs_ext2_disabled) (ty_vec128 ty))
+ (load flags @ (littleendian) addr offset)))
+ (let ((lo_addr MemArg (lower_address_bias flags addr offset 0))
+ (hi_addr MemArg (lower_address_bias flags addr offset 8))
+ (lo_val Reg (loadrev64 lo_addr))
+ (hi_val Reg (loadrev64 hi_addr)))
+ (mov_to_vec128 ty hi_val lo_val)))
;;;; Rules for `uload8` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1380,6 +2366,69 @@
(sext64_reg $I32 reg32)))
+;;;; Rules for `uloadNxM` and `sloadNxM` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Unsigned 8->16 bit extension, big-endian source value.
+(rule (lower (has_type $I16X8 (uload8x8 flags @ (bigendian) addr offset)))
+ (vec_unpacku_high $I8X16
+ (vec_load_lane_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Unsigned 8->16 bit extension, little-endian source value.
+(rule (lower (has_type $I16X8 (uload8x8 flags @ (littleendian) addr offset)))
+ (vec_unpacku_high $I8X16
+ (vec_load_lane_little_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Signed 8->16 bit extension, big-endian source value.
+(rule (lower (has_type $I16X8 (sload8x8 flags @ (bigendian) addr offset)))
+ (vec_unpacks_high $I8X16
+ (vec_load_lane_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Signed 8->16 bit extension, little-endian source value.
+(rule (lower (has_type $I16X8 (sload8x8 flags @ (littleendian) addr offset)))
+ (vec_unpacks_high $I8X16
+ (vec_load_lane_little_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Unsigned 16->32 bit extension, big-endian source value.
+(rule (lower (has_type $I32X4 (uload16x4 flags @ (bigendian) addr offset)))
+ (vec_unpacku_high $I16X8
+ (vec_load_lane_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Unsigned 16->32 bit extension, little-endian source value.
+(rule (lower (has_type $I32X4 (uload16x4 flags @ (littleendian) addr offset)))
+ (vec_unpacku_high $I16X8
+ (vec_load_lane_little_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Signed 16->32 bit extension, big-endian source value.
+(rule (lower (has_type $I32X4 (sload16x4 flags @ (bigendian) addr offset)))
+ (vec_unpacks_high $I16X8
+ (vec_load_lane_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Signed 16->32 bit extension, little-endian source value.
+(rule (lower (has_type $I32X4 (sload16x4 flags @ (littleendian) addr offset)))
+ (vec_unpacks_high $I16X8
+ (vec_load_lane_little_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Unsigned 32->64 bit extension, big-endian source value.
+(rule (lower (has_type $I64X2 (uload32x2 flags @ (bigendian) addr offset)))
+ (vec_unpacku_high $I32X4
+ (vec_load_lane_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Unsigned 32->64 bit extension, little-endian source value.
+(rule (lower (has_type $I64X2 (uload32x2 flags @ (littleendian) addr offset)))
+ (vec_unpacku_high $I32X4
+ (vec_load_lane_little_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Signed 32->64 bit extension, big-endian source value.
+(rule (lower (has_type $I64X2 (sload32x2 flags @ (bigendian) addr offset)))
+ (vec_unpacks_high $I32X4
+ (vec_load_lane_undef $I64X2 (lower_address flags addr offset) 0)))
+
+;; Signed 32->64 bit extension, little-endian source value.
+(rule (lower (has_type $I64X2 (sload32x2 flags @ (littleendian) addr offset)))
+ (vec_unpacks_high $I32X4
+ (vec_load_lane_little_undef $I64X2 (lower_address flags addr offset) 0)))
+
+
;;;; Rules for `store` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; The actual store logic for integer types is identical for the `store`,
@@ -1405,41 +2454,49 @@
(rule (lower (store flags val @ (value_type $R64) addr offset))
(side_effect (istore64_impl flags val addr offset)))
-;; Store 32-bit big-endian floating-point type.
+;; Store 32-bit big-endian floating-point type (as vector lane).
(rule (lower (store flags @ (bigendian)
val @ (value_type $F32) addr offset))
- (side_effect (fpu_store32 (put_in_reg val)
- (lower_address flags addr offset))))
+ (side_effect (vec_store_lane $F32X4 val
+ (lower_address flags addr offset) 0)))
-;; Store 32-bit little-endian floating-point type (z15 instruction).
+;; Store 32-bit little-endian floating-point type (as vector lane).
(rule (lower (store flags @ (littleendian)
- val @ (value_type (and $F32 (vxrs_ext2_enabled))) addr offset))
- (side_effect (fpu_storerev32 (put_in_reg val)
- (lower_address flags addr offset))))
+ val @ (value_type $F32) addr offset))
+ (side_effect (vec_store_lane_little $F32X4 val
+ (lower_address flags addr offset) 0)))
+
+;; Store 64-bit big-endian floating-point type (as vector lane).
+(rule (lower (store flags @ (bigendian)
+ val @ (value_type $F64) addr offset))
+ (side_effect (vec_store_lane $F64X2 val
+ (lower_address flags addr offset) 0)))
-;; Store 32-bit little-endian floating-point type (via GPR on z14).
+;; Store 64-bit little-endian floating-point type (as vector lane).
(rule (lower (store flags @ (littleendian)
- val @ (value_type (and $F32 (vxrs_ext2_disabled))) addr offset))
- (let ((gpr Reg (mov_from_fpr32 (put_in_reg val))))
- (side_effect (storerev32 gpr (lower_address flags addr offset)))))
+ val @ (value_type $F64) addr offset))
+ (side_effect (vec_store_lane_little $F64X2 val
+ (lower_address flags addr offset) 0)))
-;; Store 64-bit big-endian floating-point type.
+;; Store 128-bit big-endian vector type.
(rule (lower (store flags @ (bigendian)
- val @ (value_type $F64) addr offset))
- (side_effect (fpu_store64 (put_in_reg val)
- (lower_address flags addr offset))))
+ val @ (value_type (ty_vec128 ty)) addr offset))
+ (side_effect (vec_store val (lower_address flags addr offset))))
-;; Store 64-bit little-endian floating-point type (z15 instruction).
+;; Store 128-bit little-endian vector type (z15 instruction).
(rule (lower (store flags @ (littleendian)
- val @ (value_type (and $F64 (vxrs_ext2_enabled))) addr offset))
- (side_effect (fpu_storerev64 (put_in_reg val)
- (lower_address flags addr offset))))
+ val @ (value_type (and (ty_vec128 ty) (vxrs_ext2_enabled))) addr offset))
+ (side_effect (vec_storerev val (lower_address flags addr offset))))
-;; Store 64-bit little-endian floating-point type (via GPR on z14).
+;; Store 128-bit little-endian vector type (via GPRs on z14).
(rule (lower (store flags @ (littleendian)
- val @ (value_type (and $F64 (vxrs_ext2_disabled))) addr offset))
- (let ((gpr Reg (mov_from_fpr64 (put_in_reg val))))
- (side_effect (storerev64 gpr (lower_address flags addr offset)))))
+ val @ (value_type (and (ty_vec128 ty) (vxrs_ext2_disabled))) addr offset))
+ (let ((lo_addr MemArg (lower_address_bias flags addr offset 0))
+ (hi_addr MemArg (lower_address_bias flags addr offset 8))
+ (lo_val Reg (vec_extract_lane $I64X2 val 1 (zero_reg)))
+ (hi_val Reg (vec_extract_lane $I64X2 val 0 (zero_reg))))
+ (side_effect (side_effect_concat (storerev64 lo_val lo_addr)
+ (storerev64 hi_val hi_addr)))))
;;;; Rules for 8-bit integer stores ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1947,7 +3004,7 @@
;; Main `icmp` entry point. Generate a `ProducesBool` capturing the
;; integer comparison and immediately lower it to a 0/1 integer result.
;; In this case, it is safe to sink memory loads.
-(rule (lower (has_type ty (icmp int_cc x y)))
+(rule (lower (has_type (fits_in_64 ty) (icmp int_cc x y)))
(lower_bool ty (icmp_val $true int_cc x y)))
@@ -2033,12 +3090,36 @@
(rule (icmpu_val $true x @ (value_type (fits_in_64 ty)) (sinkable_uload32 y))
(icmpu_mem_zext32 ty x (sink_uload32 y)))
+;; Vector `icmp` produces a boolean vector.
+;; We need to handle the various IntCC flags separately here.
+
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.Equal) x y)))
+ (vec_cmpeq ty x y))
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.NotEqual) x y)))
+ (vec_not ty (vec_cmpeq ty x y)))
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.SignedGreaterThan) x y)))
+ (vec_cmph ty x y))
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.SignedLessThanOrEqual) x y)))
+ (vec_not ty (vec_cmph ty x y)))
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.SignedLessThan) x y)))
+ (vec_cmph ty y x))
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.SignedGreaterThanOrEqual) x y)))
+ (vec_not ty (vec_cmph ty y x)))
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.UnsignedGreaterThan) x y)))
+ (vec_cmphl ty x y))
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.UnsignedLessThanOrEqual) x y)))
+ (vec_not ty (vec_cmphl ty x y)))
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.UnsignedLessThan) x y)))
+ (vec_cmphl ty y x))
+(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.UnsignedGreaterThanOrEqual) x y)))
+ (vec_not ty (vec_cmphl ty y x)))
+
;;;; Rules for `fcmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Main `fcmp` entry point. Generate a `ProducesBool` capturing the
;; integer comparison and immediately lower it to a 0/1 integer result.
-(rule (lower (has_type ty (fcmp float_cc x y)))
+(rule (lower (has_type (fits_in_64 ty) (fcmp float_cc x y)))
(lower_bool ty (fcmp_val float_cc x y)))
;; Return a `ProducesBool` to implement any floating-point comparison.
@@ -2047,6 +3128,217 @@
(bool (fcmp_reg ty x y)
(floatcc_as_cond float_cc)))
+;; Vector `fcmp` produces a boolean vector.
+;; We need to handle the various FloatCC flags separately here.
+
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.Equal) x y)))
+ (vec_fcmpeq ty x y))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.NotEqual) x y)))
+ (vec_not ty (vec_fcmpeq ty x y)))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.GreaterThan) x y)))
+ (vec_fcmph ty x y))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrLessThanOrEqual) x y)))
+ (vec_not ty (vec_fcmph ty x y)))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.GreaterThanOrEqual) x y)))
+ (vec_fcmphe ty x y))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrLessThan) x y)))
+ (vec_not ty (vec_fcmphe ty x y)))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.LessThan) x y)))
+ (vec_fcmph ty y x))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) x y)))
+ (vec_not ty (vec_fcmph ty y x)))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.LessThanOrEqual) x y)))
+ (vec_fcmphe ty y x))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrGreaterThan) x y)))
+ (vec_not ty (vec_fcmphe ty y x)))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.Ordered) x y)))
+ (vec_or ty (vec_fcmphe ty x y) (vec_fcmphe ty y x)))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.Unordered) x y)))
+ (vec_not_or ty (vec_fcmphe ty x y) (vec_fcmphe ty y x)))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.OrderedNotEqual) x y)))
+ (vec_or ty (vec_fcmph ty x y) (vec_fcmph ty y x)))
+(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrEqual) x y)))
+ (vec_not_or ty (vec_fcmph ty x y) (vec_fcmph ty y x)))
+
+
+;;;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Main `vall_true` entry point. Generate a `ProducesBool` capturing the
+;; comparison and immediately lower it to a 0/1 integer result.
+(rule (lower (has_type (fits_in_64 ty) (vall_true x)))
+ (lower_bool ty (vall_true_val x)))
+
+;; Return a `ProducesBool` to implement `vall_true`.
+(decl vall_true_val (Value) ProducesBool)
+(rule (vall_true_val x @ (value_type ty))
+ (bool (vec_cmpeqs ty x (vec_imm ty 0))
+ (floatcc_as_cond (FloatCC.Unordered))))
+
+;; Short-circuit `vall_true` on the result of a `icmp`.
+(rule (vall_true_val (has_type ty (icmp (IntCC.Equal) x y)))
+ (bool (vec_cmpeqs ty x y)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (icmp (IntCC.NotEqual) x y)))
+ (bool (vec_cmpeqs ty x y)
+ (floatcc_as_cond (FloatCC.Unordered))))
+(rule (vall_true_val (has_type ty (icmp (IntCC.SignedGreaterThan) x y)))
+ (bool (vec_cmphs ty x y)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (icmp (IntCC.SignedLessThanOrEqual) x y)))
+ (bool (vec_cmphs ty x y)
+ (floatcc_as_cond (FloatCC.Unordered))))
+(rule (vall_true_val (has_type ty (icmp (IntCC.SignedLessThan) x y)))
+ (bool (vec_cmphs ty y x)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (icmp (IntCC.SignedGreaterThanOrEqual) x y)))
+ (bool (vec_cmphs ty y x)
+ (floatcc_as_cond (FloatCC.Unordered))))
+(rule (vall_true_val (has_type ty (icmp (IntCC.UnsignedGreaterThan) x y)))
+ (bool (vec_cmphls ty x y)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (icmp (IntCC.UnsignedLessThanOrEqual) x y)))
+ (bool (vec_cmphls ty x y)
+ (floatcc_as_cond (FloatCC.Unordered))))
+(rule (vall_true_val (has_type ty (icmp (IntCC.UnsignedLessThan) x y)))
+ (bool (vec_cmphls ty y x)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (icmp (IntCC.UnsignedGreaterThanOrEqual) x y)))
+ (bool (vec_cmphls ty y x)
+ (floatcc_as_cond (FloatCC.Unordered))))
+
+;; Short-circuit `vall_true` on the result of a `fcmp` where possible.
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.Equal) x y)))
+ (bool (vec_fcmpeqs ty x y)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.NotEqual) x y)))
+ (bool (vec_fcmpeqs ty x y)
+ (floatcc_as_cond (FloatCC.Unordered))))
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.GreaterThan) x y)))
+ (bool (vec_fcmphs ty x y)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.UnorderedOrLessThanOrEqual) x y)))
+ (bool (vec_fcmphs ty x y)
+ (floatcc_as_cond (FloatCC.Unordered))))
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.GreaterThanOrEqual) x y)))
+ (bool (vec_fcmphes ty x y)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.UnorderedOrLessThan) x y)))
+ (bool (vec_fcmphes ty x y)
+ (floatcc_as_cond (FloatCC.Unordered))))
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.LessThan) x y)))
+ (bool (vec_fcmphs ty y x)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) x y)))
+ (bool (vec_fcmphs ty y x)
+ (floatcc_as_cond (FloatCC.Unordered))))
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.LessThanOrEqual) x y)))
+ (bool (vec_fcmphes ty y x)
+ (floatcc_as_cond (FloatCC.Equal))))
+(rule (vall_true_val (has_type ty (fcmp (FloatCC.UnorderedOrGreaterThan) x y)))
+ (bool (vec_fcmphes ty y x)
+ (floatcc_as_cond (FloatCC.Unordered))))
+
+
+;;;; Rules for `vany_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Main `vany_true` entry point. Generate a `ProducesBool` capturing the
+;; comparison and immediately lower it to a 0/1 integer result.
+(rule (lower (has_type (fits_in_64 ty) (vany_true x)))
+ (lower_bool ty (vany_true_val x)))
+
+;; Return a `ProducesBool` to implement `vany_true`.
+(decl vany_true_val (Value) ProducesBool)
+(rule (vany_true_val x @ (value_type ty))
+ (bool (vec_cmpeqs ty x (vec_imm ty 0))
+ (floatcc_as_cond (FloatCC.NotEqual))))
+
+;; Short-circuit `vany_true` on the result of a `icmp`.
+(rule (vany_true_val (has_type ty (icmp (IntCC.Equal) x y)))
+ (bool (vec_cmpeqs ty x y)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (icmp (IntCC.NotEqual) x y)))
+ (bool (vec_cmpeqs ty x y)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+(rule (vany_true_val (has_type ty (icmp (IntCC.SignedGreaterThan) x y)))
+ (bool (vec_cmphs ty x y)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (icmp (IntCC.SignedLessThanOrEqual) x y)))
+ (bool (vec_cmphs ty x y)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+(rule (vany_true_val (has_type ty (icmp (IntCC.SignedLessThan) x y)))
+ (bool (vec_cmphs ty y x)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (icmp (IntCC.SignedGreaterThanOrEqual) x y)))
+ (bool (vec_cmphs ty y x)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+(rule (vany_true_val (has_type ty (icmp (IntCC.UnsignedGreaterThan) x y)))
+ (bool (vec_cmphls ty x y)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (icmp (IntCC.UnsignedLessThanOrEqual) x y)))
+ (bool (vec_cmphls ty x y)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+(rule (vany_true_val (has_type ty (icmp (IntCC.UnsignedLessThan) x y)))
+ (bool (vec_cmphls ty y x)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (icmp (IntCC.UnsignedGreaterThanOrEqual) x y)))
+ (bool (vec_cmphls ty y x)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+
+;; Short-circuit `vany_true` on the result of a `fcmp` where possible.
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.Equal) x y)))
+ (bool (vec_fcmpeqs ty x y)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.NotEqual) x y)))
+ (bool (vec_fcmpeqs ty x y)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.GreaterThan) x y)))
+ (bool (vec_fcmphs ty x y)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.UnorderedOrLessThanOrEqual) x y)))
+ (bool (vec_fcmphs ty x y)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.GreaterThanOrEqual) x y)))
+ (bool (vec_fcmphes ty x y)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.UnorderedOrLessThan) x y)))
+ (bool (vec_fcmphes ty x y)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.LessThan) x y)))
+ (bool (vec_fcmphs ty y x)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) x y)))
+ (bool (vec_fcmphs ty y x)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.LessThanOrEqual) x y)))
+ (bool (vec_fcmphes ty y x)
+ (floatcc_as_cond (FloatCC.Ordered))))
+(rule (vany_true_val (has_type ty (fcmp (FloatCC.UnorderedOrGreaterThan) x y)))
+ (bool (vec_fcmphes ty y x)
+ (floatcc_as_cond (FloatCC.NotEqual))))
+
+
+;;;; Rules for `vhigh_bits` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(rule (lower (vhigh_bits x @ (value_type (multi_lane 8 16))))
+ (let ((mask Reg (vec_imm $I8X16 (imm8x16 0 8 16 24 32 40 48 56
+ 64 72 80 88 96 104 112 120))))
+ (vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))
+
+(rule (lower (vhigh_bits x @ (value_type (multi_lane 16 8))))
+ (let ((mask Reg (vec_imm $I8X16 (imm8x16 128 128 128 128 128 128 128 128
+ 0 16 32 48 64 80 96 112))))
+ (vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))
+
+(rule (lower (vhigh_bits x @ (value_type (multi_lane 32 4))))
+ (let ((mask Reg (vec_imm $I8X16 (imm8x16 128 128 128 128 128 128 128 128
+ 128 128 128 128 0 32 64 96))))
+ (vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))
+
+(rule (lower (vhigh_bits x @ (value_type (multi_lane 64 2))))
+ (let ((mask Reg (vec_imm $I8X16 (imm8x16 128 128 128 128 128 128 128 128
+ 128 128 128 128 128 128 0 64))))
+ (vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))
+
;;;; Rules for `is_null` and `is_invalid` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/s390x/lower.rs b/cranelift/codegen/src/isa/s390x/lower.rs
index 2c87621aae32..df93c47023dd 100644
--- a/cranelift/codegen/src/isa/s390x/lower.rs
+++ b/cranelift/codegen/src/isa/s390x/lower.rs
@@ -43,15 +43,28 @@ impl LowerBackend for S390xBackend {
| Opcode::Bconst
| Opcode::F32const
| Opcode::F64const
+ | Opcode::Vconst
| Opcode::Null
| Opcode::Iadd
| Opcode::IaddIfcout
| Opcode::Isub
+ | Opcode::UaddSat
+ | Opcode::SaddSat
+ | Opcode::UsubSat
+ | Opcode::SsubSat
+ | Opcode::IaddPairwise
+ | Opcode::Imin
+ | Opcode::Umin
+ | Opcode::Imax
+ | Opcode::Umax
+ | Opcode::AvgRound
| Opcode::Iabs
| Opcode::Ineg
| Opcode::Imul
| Opcode::Umulhi
| Opcode::Smulhi
+ | Opcode::WideningPairwiseDotProductS
+ | Opcode::SqmulRoundSat
| Opcode::Udiv
| Opcode::Urem
| Opcode::Sdiv
@@ -64,6 +77,13 @@ impl LowerBackend for S390xBackend {
| Opcode::Ireduce
| Opcode::Uextend
| Opcode::Sextend
+ | Opcode::Snarrow
+ | Opcode::Unarrow
+ | Opcode::Uunarrow
+ | Opcode::SwidenLow
+ | Opcode::SwidenHigh
+ | Opcode::UwidenLow
+ | Opcode::UwidenHigh
| Opcode::Bnot
| Opcode::Band
| Opcode::Bor
@@ -72,6 +92,7 @@ impl LowerBackend for S390xBackend {
| Opcode::BorNot
| Opcode::BxorNot
| Opcode::Bitselect
+ | Opcode::Vselect
| Opcode::Breduce
| Opcode::Bextend
| Opcode::Bmask
@@ -86,11 +107,15 @@ impl LowerBackend for S390xBackend {
| Opcode::Fdiv
| Opcode::Fmin
| Opcode::Fmax
+ | Opcode::FminPseudo
+ | Opcode::FmaxPseudo
| Opcode::Sqrt
| Opcode::Fneg
| Opcode::Fabs
| Opcode::Fpromote
| Opcode::Fdemote
+ | Opcode::FvpromoteLow
+ | Opcode::Fvdemote
| Opcode::Ceil
| Opcode::Floor
| Opcode::Trunc
@@ -99,11 +124,20 @@ impl LowerBackend for S390xBackend {
| Opcode::Fcopysign
| Opcode::FcvtFromUint
| Opcode::FcvtFromSint
+ | Opcode::FcvtLowFromSint
| Opcode::FcvtToUint
| Opcode::FcvtToSint
| Opcode::FcvtToUintSat
| Opcode::FcvtToSintSat
+ | Opcode::Splat
+ | Opcode::Swizzle
+ | Opcode::Shuffle
+ | Opcode::Insertlane
+ | Opcode::Extractlane
+ | Opcode::ScalarToVector
+ | Opcode::VhighBits
| Opcode::Bitcast
+ | Opcode::RawBitcast
| Opcode::Load
| Opcode::Uload8
| Opcode::Sload8
@@ -111,6 +145,12 @@ impl LowerBackend for S390xBackend {
| Opcode::Sload16
| Opcode::Uload32
| Opcode::Sload32
+ | Opcode::Uload8x8
+ | Opcode::Sload8x8
+ | Opcode::Uload16x4
+ | Opcode::Sload16x4
+ | Opcode::Uload32x2
+ | Opcode::Sload32x2
| Opcode::Store
| Opcode::Istore8
| Opcode::Istore16
@@ -122,6 +162,8 @@ impl LowerBackend for S390xBackend {
| Opcode::Fence
| Opcode::Icmp
| Opcode::Fcmp
+ | Opcode::VanyTrue
+ | Opcode::VallTrue
| Opcode::IsNull
| Opcode::IsInvalid
| Opcode::Select
@@ -147,57 +189,15 @@ impl LowerBackend for S390xBackend {
)
}
- Opcode::UaddSat
- | Opcode::SaddSat
- | Opcode::UsubSat
- | Opcode::SsubSat
- | Opcode::Bitrev
- | Opcode::FcvtLowFromSint
+ Opcode::Bitrev
| Opcode::ConstAddr
| Opcode::TlsValue
| Opcode::GetPinnedReg
| Opcode::SetPinnedReg
| Opcode::Isplit
| Opcode::Iconcat
- | Opcode::RawBitcast
- | Opcode::Splat
- | Opcode::Swizzle
- | Opcode::Insertlane
- | Opcode::Extractlane
- | Opcode::Imin
- | Opcode::Umin
- | Opcode::Imax
- | Opcode::Umax
- | Opcode::AvgRound
- | Opcode::FminPseudo
- | Opcode::FmaxPseudo
- | Opcode::Uload8x8
- | Opcode::Sload8x8
- | Opcode::Uload16x4
- | Opcode::Sload16x4
- | Opcode::Uload32x2
- | Opcode::Sload32x2
- | Opcode::Vconst
- | Opcode::Shuffle
| Opcode::Vsplit
| Opcode::Vconcat
- | Opcode::Vselect
- | Opcode::VanyTrue
- | Opcode::VallTrue
- | Opcode::VhighBits
- | Opcode::ScalarToVector
- | Opcode::Snarrow
- | Opcode::Unarrow
- | Opcode::Uunarrow
- | Opcode::SwidenLow
- | Opcode::SwidenHigh
- | Opcode::UwidenLow
- | Opcode::UwidenHigh
- | Opcode::WideningPairwiseDotProductS
- | Opcode::SqmulRoundSat
- | Opcode::FvpromoteLow
- | Opcode::Fvdemote
- | Opcode::IaddPairwise
| Opcode::DynamicStackLoad
| Opcode::DynamicStackStore
| Opcode::DynamicStackAddr
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs
index 2d41c6a88adc..4db95d40afb5 100644
--- a/cranelift/codegen/src/isa/s390x/lower/isle.rs
+++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs
@@ -6,7 +6,7 @@ pub mod generated_code;
// Types that the generated ISLE code uses via `use super::*`.
use crate::isa::s390x::abi::S390xMachineDeps;
use crate::isa::s390x::inst::{
- stack_reg, writable_gpr, zero_reg, CallIndInfo, CallInfo, Cond, Inst as MInst, MemArg,
+ stack_reg, writable_gpr, zero_reg, CallIndInfo, CallInfo, Cond, Inst as MInst, MemArg, UImm12,
UImm16Shifted, UImm32Shifted,
};
use crate::isa::s390x::settings::Flags as IsaFlags;
@@ -91,6 +91,8 @@ where
defs,
clobbers,
opcode: *opcode,
+ caller_callconv: self.lower_ctx.abi().call_conv(),
+ callee_callconv: abi.call_conv(),
})
}
@@ -102,6 +104,8 @@ where
defs,
clobbers,
opcode: *opcode,
+ caller_callconv: self.lower_ctx.abi().call_conv(),
+ callee_callconv: abi.call_conv(),
})
}
@@ -195,6 +199,46 @@ where
}
}
+ #[inline]
+ fn u64_pair_split(&mut self, n: u128) -> (u64, u64) {
+ ((n >> 64) as u64, n as u64)
+ }
+
+ #[inline]
+ fn u64_pair_concat(&mut self, hi: u64, lo: u64) -> u128 {
+ (hi as u128) << 64 | (lo as u128)
+ }
+
+ #[inline]
+ fn u32_pair_split(&mut self, n: u64) -> (u32, u32) {
+ ((n >> 32) as u32, n as u32)
+ }
+
+ #[inline]
+ fn u32_pair_concat(&mut self, hi: u32, lo: u32) -> u64 {
+ (hi as u64) << 32 | (lo as u64)
+ }
+
+ #[inline]
+ fn u16_pair_split(&mut self, n: u32) -> (u16, u16) {
+ ((n >> 16) as u16, n as u16)
+ }
+
+ #[inline]
+ fn u16_pair_concat(&mut self, hi: u16, lo: u16) -> u32 {
+ (hi as u32) << 16 | (lo as u32)
+ }
+
+ #[inline]
+ fn u8_pair_split(&mut self, n: u16) -> (u8, u8) {
+ ((n >> 8) as u8, n as u8)
+ }
+
+ #[inline]
+ fn u8_pair_concat(&mut self, hi: u8, lo: u8) -> u16 {
+ (hi as u16) << 8 | (lo as u16)
+ }
+
#[inline]
fn u8_as_u16(&mut self, n: u8) -> u16 {
n as u16
@@ -248,6 +292,15 @@ where
}
}
+ #[inline]
+ fn i16_from_u32(&mut self, n: u32) -> Option<i16> {
+ if let Ok(imm) = i16::try_from(n as i32) {
+ Some(imm)
+ } else {
+ None
+ }
+ }
+
#[inline]
fn uimm32shifted_from_u64(&mut self, n: u64) -> Option<UImm32Shifted> {
UImm32Shifted::maybe_from_u64(n)
@@ -258,11 +311,49 @@ where
UImm16Shifted::maybe_from_u64(n)
}
+ #[inline]
+ fn be_lane_idx(&mut self, ty: Type, idx: u8) -> u8 {
+ ty.lane_count() as u8 - 1 - idx
+ }
+
+ #[inline]
+ fn lane_byte_mask(&mut self, ty: Type, idx: u8) -> u16 {
+ let lane_bytes = (ty.lane_bits() / 8) as u8;
+ let lane_mask = (1u16 << lane_bytes) - 1;
+ lane_mask << (16 - ((idx + 1) * lane_bytes))
+ }
+
+ #[inline]
+ fn shuffle_mask_from_u128(&mut self, idx: u128) -> (u128, u16) {
+ let bytes = idx.to_be_bytes();
+ let and_mask = bytes.iter().fold(0, |acc, &x| (acc << 1) | (x < 32) as u16);
+ let bytes = bytes.map(|x| {
+ if x < 16 {
+ 15 - x
+ } else if x < 32 {
+ 47 - x
+ } else {
+ 128
+ }
+ });
+ let permute_mask = u128::from_be_bytes(bytes);
+ (permute_mask, and_mask)
+ }
+
#[inline]
fn u64_from_value(&mut self, val: Value) -> Option<u64> {
let inst = self.lower_ctx.dfg().value_def(val).inst()?;
let constant = self.lower_ctx.get_constant(inst)?;
- Some(constant)
+ let ty = self.lower_ctx.output_ty(inst, 0);
+ Some(zero_extend_to_u64(constant, self.ty_bits(ty).unwrap()))
+ }
+
+ #[inline]
+ fn u64_from_inverted_value(&mut self, val: Value) -> Option<u64> {
+ let inst = self.lower_ctx.dfg().value_def(val).inst()?;
+ let constant = self.lower_ctx.get_constant(inst)?;
+ let ty = self.lower_ctx.output_ty(inst, 0);
+ Some(zero_extend_to_u64(!constant, self.ty_bits(ty).unwrap()))
}
#[inline]
@@ -349,22 +440,22 @@ where
#[inline]
fn uimm16shifted_from_inverted_value(&mut self, val: Value) -> Option<UImm16Shifted> {
- let constant = self.u64_from_value(val)?;
- let imm = UImm16Shifted::maybe_from_u64(!constant)?;
+ let constant = self.u64_from_inverted_value(val)?;
+ let imm = UImm16Shifted::maybe_from_u64(constant)?;
Some(imm.negate_bits())
}
#[inline]
fn uimm32shifted_from_inverted_value(&mut self, val: Value) -> Option<UImm32Shifted> {
- let constant = self.u64_from_value(val)?;
- let imm = UImm32Shifted::maybe_from_u64(!constant)?;
+ let constant = self.u64_from_inverted_value(val)?;
+ let imm = UImm32Shifted::maybe_from_u64(constant)?;
Some(imm.negate_bits())
}
#[inline]
fn mask_amt_imm(&mut self, ty: Type, amt: i64) -> u8 {
- let mask = self.ty_bits(ty).unwrap() - 1;
- (amt as u8) & mask
+ let mask = ty.lane_bits() - 1;
+ (amt as u8) & (mask as u8)
}
#[inline]
@@ -498,13 +589,18 @@ where
}
#[inline]
- fn memarg_reg_plus_reg(&mut self, x: Reg, y: Reg, flags: MemFlags) -> MemArg {
- MemArg::reg_plus_reg(x, y, flags)
+ fn memarg_reg_plus_reg(&mut self, x: Reg, y: Reg, bias: u8, flags: MemFlags) -> MemArg {
+ MemArg::BXD12 {
+ base: x,
+ index: y,
+ disp: UImm12::maybe_from_u64(bias as u64).unwrap(),
+ flags,
+ }
}
#[inline]
- fn memarg_reg_plus_off(&mut self, reg: Reg, off: i64, flags: MemFlags) -> MemArg {
- MemArg::reg_plus_off(reg, off, flags)
+ fn memarg_reg_plus_off(&mut self, reg: Reg, off: i64, bias: u8, flags: MemFlags) -> MemArg {
+ MemArg::reg_plus_off(reg, off + (bias as i64), flags)
}
#[inline]
@@ -586,6 +682,17 @@ where
}
}
+/// Zero-extend the low `from_bits` bits of `value` to a full u64.
+#[inline]
+fn zero_extend_to_u64(value: u64, from_bits: u8) -> u64 {
+ assert!(from_bits <= 64);
+ if from_bits >= 64 {
+ value
+ } else {
+ value & ((1u64 << from_bits) - 1)
+ }
+}
+
/// Sign-extend the low `from_bits` bits of `value` to a full u64.
#[inline]
fn sign_extend_to_u64(value: u64, from_bits: u8) -> u64 {
diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs
index 45bf8884b916..718871e82708 100644
--- a/cranelift/codegen/src/machinst/abi_impl.rs
+++ b/cranelift/codegen/src/machinst/abi_impl.rs
@@ -696,6 +696,11 @@ impl ABISig {
let ret_arg = self.stack_ret_arg?;
Some(self.args[ret_arg].clone())
}
+
+ /// Get calling convention used.
+ pub fn call_conv(&self) -> isa::CallConv {
+ self.call_conv
+ }
}
/// ABI object for a function body.
diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs
index 28005863df33..c1bdc418bd6a 100644
--- a/cranelift/codegen/src/machinst/isle.rs
+++ b/cranelift/codegen/src/machinst/isle.rs
@@ -7,7 +7,8 @@ use std::cell::Cell;
pub use super::MachLabel;
pub use crate::ir::{
- ArgumentExtension, DynamicStackSlot, ExternalName, FuncRef, GlobalValue, SigRef, StackSlot,
+ ArgumentExtension, Constant, DynamicStackSlot, ExternalName, FuncRef, GlobalValue, Immediate,
+ SigRef, StackSlot,
};
pub use crate::isa::unwind::UnwindInst;
pub use crate::machinst::{ABIArg, ABIArgSlot, ABISig, RealReg, Reg, RelocDistance, Writable};
@@ -539,6 +540,18 @@ macro_rules! isle_prelude_methods {
}
}
+ #[inline]
+ fn u128_from_immediate(&mut self, imm: Immediate) -> Option<u128> {
+ let bytes = self.lower_ctx.get_immediate_data(imm).as_slice();
+ Some(u128::from_le_bytes(bytes.try_into().ok()?))
+ }
+
+ #[inline]
+ fn u128_from_constant(&mut self, constant: Constant) -> Option<u128> {
+ let bytes = self.lower_ctx.get_constant_data(constant).as_slice();
+ Some(u128::from_le_bytes(bytes.try_into().ok()?))
+ }
+
fn nonzero_u64_from_imm64(&mut self, val: Imm64) -> Option<u64> {
match val.bits() {
0 => None,
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index a496f9a61657..42a4f74c240b 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -12,8 +12,8 @@ use crate::inst_predicates::{has_lowering_side_effect, is_constant_64bit};
use crate::ir::{
types::{FFLAGS, IFLAGS},
ArgumentPurpose, Block, Constant, ConstantData, DataFlowGraph, ExternalName, Function,
- GlobalValue, GlobalValueData, Inst, InstructionData, MemFlags, Opcode, Signature, SourceLoc,
- Type, Value, ValueDef, ValueLabelAssignments, ValueLabelStart,
+ GlobalValue, GlobalValueData, Immediate, Inst, InstructionData, MemFlags, Opcode, Signature,
+ SourceLoc, Type, Value, ValueDef, ValueLabelAssignments, ValueLabelStart,
};
use crate::machinst::{
non_writable_value_regs, writable_value_regs, ABICallee, BlockIndex, BlockLoweringOrder,
@@ -167,6 +167,8 @@ pub trait LowerCtx {
/// for the input produced by the sunk instruction), otherwise the
/// side-effect will occur twice.
fn sink_inst(&mut self, ir_inst: Inst);
+ /// Retrieve immediate data given a handle.
+ fn get_immediate_data(&self, imm: Immediate) -> &ConstantData;
/// Retrieve constant data given a handle.
fn get_constant_data(&self, constant_handle: Constant) -> &ConstantData;
/// Indicate that a constant should be emitted.
@@ -1448,6 +1450,10 @@ impl<'func, I: VCodeInst> LowerCtx for Lower<'func, I> {
self.inst_sunk.insert(ir_inst);
}
+ fn get_immediate_data(&self, imm: Immediate) -> &ConstantData {
+ self.f.dfg.immediates.get(imm).unwrap()
+ }
+
fn get_constant_data(&self, constant_handle: Constant) -> &ConstantData {
self.f.dfg.constants.get(constant_handle)
}
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index ccaef3234197..abbac35bb922 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -657,6 +657,17 @@
(decl reloc_distance_near () RelocDistance)
(extern extractor reloc_distance_near reloc_distance_near)
+;; Accessor for `Immediate` as u128.
+
+(decl u128_from_immediate (u128) Immediate)
+(extern extractor u128_from_immediate u128_from_immediate)
+
+;; Accessor for `Constant` as u128.
+
+(decl u128_from_constant (u128) Constant)
+(extern extractor u128_from_constant u128_from_constant)
+
+
;;;; Helpers for tail recursion loops ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; A range of integers to loop through.
diff --git a/cranelift/filetests/filetests/isa/s390x/condops.clif b/cranelift/filetests/filetests/isa/s390x/condops.clif
index 5a73e0ae1e31..d84cd49c0500 100644
--- a/cranelift/filetests/filetests/isa/s390x/condops.clif
+++ b/cranelift/filetests/filetests/isa/s390x/condops.clif
@@ -43,3 +43,18 @@ block0(v0: i32, v1: i8, v2: i8):
; locre %r2, %r3
; br %r14
+function %i(i32, i8x16, i8x16) -> i8x16 {
+block0(v0: i32, v1: i8x16, v2: i8x16):
+ v3 = iconst.i32 42
+ v4 = icmp.i32 eq v0, v3
+ v5 = select.i8x16 v4, v1, v2
+ return v5
+}
+
+; block0:
+; vlr %v20, %v24
+; clfi %r2, 42
+; vlr %v24, %v25
+; jne 10 ; vlr %v24, %v20
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/floating-point.clif b/cranelift/filetests/filetests/isa/s390x/floating-point.clif
index 4a8a84f37e6c..47e28b87d687 100644
--- a/cranelift/filetests/filetests/isa/s390x/floating-point.clif
+++ b/cranelift/filetests/filetests/isa/s390x/floating-point.clif
@@ -168,6 +168,46 @@ block0(v0: f64, v1: f64):
; wfmaxdb %f0, %f0, %f2, 1
; br %r14
+function %fmin_pseudo_f32(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+ v2 = fmin_pseudo v0, v1
+ return v2
+}
+
+; block0:
+; wfminsb %f0, %f0, %f2, 3
+; br %r14
+
+function %fmin_pseudo_f64(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+ v2 = fmin_pseudo v0, v1
+ return v2
+}
+
+; block0:
+; wfmindb %f0, %f0, %f2, 3
+; br %r14
+
+function %fmax_pseudo_f32(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+ v2 = fmax_pseudo v0, v1
+ return v2
+}
+
+; block0:
+; wfmaxsb %f0, %f0, %f2, 3
+; br %r14
+
+function %fmax_pseudo_f64(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+ v2 = fmax_pseudo v0, v1
+ return v2
+}
+
+; block0:
+; wfmaxdb %f0, %f0, %f2, 3
+; br %r14
+
function %sqrt_f32(f32) -> f32 {
block0(v0: f32):
v1 = sqrt v0
diff --git a/cranelift/filetests/filetests/isa/s390x/fpmem-arch13.clif b/cranelift/filetests/filetests/isa/s390x/fpmem-arch13.clif
index 76224768bc9f..736d72b7a1da 100644
--- a/cranelift/filetests/filetests/isa/s390x/fpmem-arch13.clif
+++ b/cranelift/filetests/filetests/isa/s390x/fpmem-arch13.clif
@@ -8,7 +8,7 @@ block0(v0: i64):
}
; block0:
-; vlebrg %f0, 0(%r2), 0
+; vlebrg %v0, 0(%r2), 0
; br %r14
function %load_f32_little(i64) -> f32 {
@@ -18,7 +18,7 @@ block0(v0: i64):
}
; block0:
-; vlebrf %f0, 0(%r2), 0
+; vlebrf %v0, 0(%r2), 0
; br %r14
function %store_f64_little(f64, i64) {
@@ -28,7 +28,7 @@ block0(v0: f64, v1: i64):
}
; block0:
-; vstebrg %f0, 0(%r2), 0
+; vstebrg %v0, 0(%r2), 0
; br %r14
function %store_f32_little(f32, i64) {
@@ -38,6 +38,6 @@ block0(v0: f32, v1: i64):
}
; block0:
-; vstebrf %f0, 0(%r2), 0
+; vstebrf %v0, 0(%r2), 0
; br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-arithmetic.clif b/cranelift/filetests/filetests/isa/s390x/vec-arithmetic.clif
new file mode 100644
index 000000000000..334c43821b8a
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-arithmetic.clif
@@ -0,0 +1,824 @@
+test compile precise-output
+target s390x
+
+function %iadd_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = iadd.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vag %v24, %v24, %v25
+; br %r14
+
+function %iadd_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = iadd.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vaf %v24, %v24, %v25
+; br %r14
+
+function %iadd_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = iadd.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vah %v24, %v24, %v25
+; br %r14
+
+function %iadd_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = iadd.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vab %v24, %v24, %v25
+; br %r14
+
+function %isub_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = isub.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vsg %v24, %v24, %v25
+; br %r14
+
+function %isub_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = isub.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vsf %v24, %v24, %v25
+; br %r14
+
+function %isub_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = isub.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vsh %v24, %v24, %v25
+; br %r14
+
+function %isub_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = isub.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vsb %v24, %v24, %v25
+; br %r14
+
+function %iabs_i64x2(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = iabs.i64x2 v0
+ return v1
+}
+
+; block0:
+; vlpg %v24, %v24
+; br %r14
+
+function %iabs_i32x4(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = iabs.i32x4 v0
+ return v1
+}
+
+; block0:
+; vlpf %v24, %v24
+; br %r14
+
+function %iabs_i16x8(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = iabs.i16x8 v0
+ return v1
+}
+
+; block0:
+; vlph %v24, %v24
+; br %r14
+
+function %iabs_i8x16(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = iabs.i8x16 v0
+ return v1
+}
+
+; block0:
+; vlpb %v24, %v24
+; br %r14
+
+function %ineg_i64x2(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = ineg.i64x2 v0
+ return v1
+}
+
+; block0:
+; vlcg %v24, %v24
+; br %r14
+
+function %ineg_i32x4(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = ineg.i32x4 v0
+ return v1
+}
+
+; block0:
+; vlcf %v24, %v24
+; br %r14
+
+function %ineg_i16x8(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = ineg.i16x8 v0
+ return v1
+}
+
+; block0:
+; vlch %v24, %v24
+; br %r14
+
+function %ineg_i8x16(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = ineg.i8x16 v0
+ return v1
+}
+
+; block0:
+; vlcb %v24, %v24
+; br %r14
+
+function %umax_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = umax.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vmxlg %v24, %v24, %v25
+; br %r14
+
+function %umax_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = umax.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vmxlf %v24, %v24, %v25
+; br %r14
+
+function %umax_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = umax.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vmxlh %v24, %v24, %v25
+; br %r14
+
+function %umax_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = umax.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vmxlb %v24, %v24, %v25
+; br %r14
+
+function %umin_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = umin.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vmnlg %v24, %v24, %v25
+; br %r14
+
+function %umin_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = umin.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vmnlf %v24, %v24, %v25
+; br %r14
+
+function %umin_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = umin.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vmnlh %v24, %v24, %v25
+; br %r14
+
+function %umin_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = umin.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vmnlb %v24, %v24, %v25
+; br %r14
+
+function %imax_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = imax.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vmxg %v24, %v24, %v25
+; br %r14
+
+function %imax_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = imax.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vmxf %v24, %v24, %v25
+; br %r14
+
+function %imax_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = imax.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vmxh %v24, %v24, %v25
+; br %r14
+
+function %imax_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = imax.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vmxb %v24, %v24, %v25
+; br %r14
+
+function %imin_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = imin.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vmng %v24, %v24, %v25
+; br %r14
+
+function %imin_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = imin.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vmnf %v24, %v24, %v25
+; br %r14
+
+function %imin_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = imin.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vmnh %v24, %v24, %v25
+; br %r14
+
+function %imin_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = imin.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vmnb %v24, %v24, %v25
+; br %r14
+
+function %avg_round_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = avg_round.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vavglg %v24, %v24, %v25
+; br %r14
+
+function %avg_round_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = avg_round.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vavglf %v24, %v24, %v25
+; br %r14
+
+function %avg_round_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = avg_round.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vavglh %v24, %v24, %v25
+; br %r14
+
+function %avg_round_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = avg_round.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vavglb %v24, %v24, %v25
+; br %r14
+
+function %uadd_sat64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = uadd_sat.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vag %v5, %v24, %v25
+; vchlg %v7, %v24, %v5
+; vo %v24, %v5, %v7
+; br %r14
+
+function %uadd_sat32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = uadd_sat.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vaf %v5, %v24, %v25
+; vchlf %v7, %v24, %v5
+; vo %v24, %v5, %v7
+; br %r14
+
+function %uadd_sat16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = uadd_sat.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vah %v5, %v24, %v25
+; vchlh %v7, %v24, %v5
+; vo %v24, %v5, %v7
+; br %r14
+
+function %uadd_sat8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = uadd_sat.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vab %v5, %v24, %v25
+; vchlb %v7, %v24, %v5
+; vo %v24, %v5, %v7
+; br %r14
+
+function %sadd_sat32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = sadd_sat.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vuphf %v5, %v24
+; vuphf %v7, %v25
+; vag %v17, %v5, %v7
+; vuplf %v19, %v24
+; vuplf %v21, %v25
+; vag %v23, %v19, %v21
+; vpksg %v24, %v17, %v23
+; br %r14
+
+function %sadd_sat16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = sadd_sat.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vuphh %v5, %v24
+; vuphh %v7, %v25
+; vaf %v17, %v5, %v7
+; vuplh %v19, %v24
+; vuplh %v21, %v25
+; vaf %v23, %v19, %v21
+; vpksf %v24, %v17, %v23
+; br %r14
+
+function %sadd_sat8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = sadd_sat.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vuphb %v5, %v24
+; vuphb %v7, %v25
+; vah %v17, %v5, %v7
+; vuplb %v19, %v24
+; vuplb %v21, %v25
+; vah %v23, %v19, %v21
+; vpksh %v24, %v17, %v23
+; br %r14
+
+function %iadd_pairwise_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = iadd_pairwise.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vrepib %v5, 32
+; vsrlb %v7, %v25, %v5
+; vaf %v17, %v25, %v7
+; vsrlb %v19, %v24, %v5
+; vaf %v21, %v24, %v19
+; vpkg %v24, %v17, %v21
+; br %r14
+
+function %usub_sat64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = usub_sat.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vsg %v5, %v24, %v25
+; vchlg %v7, %v24, %v25
+; vn %v24, %v5, %v7
+; br %r14
+
+function %usub_sat32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = usub_sat.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vsf %v5, %v24, %v25
+; vchlf %v7, %v24, %v25
+; vn %v24, %v5, %v7
+; br %r14
+
+function %usub_sat16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = usub_sat.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vsh %v5, %v24, %v25
+; vchlh %v7, %v24, %v25
+; vn %v24, %v5, %v7
+; br %r14
+
+function %usub_sat8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = usub_sat.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vsb %v5, %v24, %v25
+; vchlb %v7, %v24, %v25
+; vn %v24, %v5, %v7
+; br %r14
+
+function %ssub_sat32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = ssub_sat.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vuphf %v5, %v24
+; vuphf %v7, %v25
+; vsg %v17, %v5, %v7
+; vuplf %v19, %v24
+; vuplf %v21, %v25
+; vsg %v23, %v19, %v21
+; vpksg %v24, %v17, %v23
+; br %r14
+
+function %ssub_sat16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = ssub_sat.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vuphh %v5, %v24
+; vuphh %v7, %v25
+; vsf %v17, %v5, %v7
+; vuplh %v19, %v24
+; vuplh %v21, %v25
+; vsf %v23, %v19, %v21
+; vpksf %v24, %v17, %v23
+; br %r14
+
+function %ssub_sat8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = ssub_sat.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vuphb %v5, %v24
+; vuphb %v7, %v25
+; vsh %v17, %v5, %v7
+; vuplb %v19, %v24
+; vuplb %v21, %v25
+; vsh %v23, %v19, %v21
+; vpksh %v24, %v17, %v23
+; br %r14
+
+function %iadd_pairwise_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = iadd_pairwise.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vrepib %v5, 32
+; vsrlb %v7, %v25, %v5
+; vaf %v17, %v25, %v7
+; vsrlb %v19, %v24, %v5
+; vaf %v21, %v24, %v19
+; vpkg %v24, %v17, %v21
+; br %r14
+
+function %iadd_pairwise_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = iadd_pairwise.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vrepib %v5, 16
+; vsrlb %v7, %v25, %v5
+; vah %v17, %v25, %v7
+; vsrlb %v19, %v24, %v5
+; vah %v21, %v24, %v19
+; vpkf %v24, %v17, %v21
+; br %r14
+
+function %iadd_pairwise_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = iadd_pairwise.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vrepib %v5, 8
+; vsrlb %v7, %v25, %v5
+; vab %v17, %v25, %v7
+; vsrlb %v19, %v24, %v5
+; vab %v21, %v24, %v19
+; vpkh %v24, %v17, %v21
+; br %r14
+
+function %imul_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = imul.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vlgvg %r3, %v24, 0
+; vlgvg %r5, %v25, 0
+; msgr %r3, %r5
+; vlgvg %r5, %v24, 1
+; vlgvg %r4, %v25, 1
+; msgr %r5, %r4
+; vlvgp %v24, %r3, %r5
+; br %r14
+
+function %imul_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = imul.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vmlf %v24, %v24, %v25
+; br %r14
+
+function %imul_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = imul.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vmlhw %v24, %v24, %v25
+; br %r14
+
+function %imul_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = imul.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vmlb %v24, %v24, %v25
+; br %r14
+
+function %umulhi_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = umulhi.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vlgvg %r3, %v24, 0
+; vlgvg %r1, %v25, 0
+; mlgr %r0, %r3
+; lgr %r2, %r0
+; vlgvg %r3, %v24, 1
+; vlgvg %r1, %v25, 1
+; mlgr %r0, %r3
+; vlvgp %v24, %r2, %r0
+; br %r14
+
+function %umulhi_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = umulhi.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vmlhf %v24, %v24, %v25
+; br %r14
+
+function %umulhi_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = umulhi.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vmlhh %v24, %v24, %v25
+; br %r14
+
+function %umulhi_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = umulhi.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vmlhb %v24, %v24, %v25
+; br %r14
+
+function %smulhi_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = smulhi.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vlgvg %r3, %v24, 0
+; vlgvg %r5, %v25, 0
+; mgrk %r0, %r3, %r5
+; lgr %r3, %r0
+; vlgvg %r2, %v24, 1
+; vlgvg %r4, %v25, 1
+; mgrk %r0, %r2, %r4
+; lgr %r4, %r3
+; vlvgp %v24, %r4, %r0
+; br %r14
+
+function %smulhi_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = smulhi.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vmhf %v24, %v24, %v25
+; br %r14
+
+function %smulhi_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = smulhi.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vmhh %v24, %v24, %v25
+; br %r14
+
+function %smulhi_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = smulhi.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vmhb %v24, %v24, %v25
+; br %r14
+
+function %widening_pairwise_dot_product_s_i16x8(i16x8, i16x8) -> i32x4 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = widening_pairwise_dot_product_s v0, v1
+ return v2
+}
+
+; block0:
+; vmeh %v5, %v24, %v25
+; vmoh %v7, %v24, %v25
+; vaf %v24, %v5, %v7
+; br %r14
+
+function %sqmul_round_sat(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = sqmul_round_sat.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vuphh %v5, %v24
+; vuphh %v7, %v25
+; vmlf %v17, %v5, %v7
+; vgmf %v19, 17, 17
+; vaf %v21, %v17, %v19
+; vesraf %v23, %v21, 15
+; vuplh %v26, %v24
+; vuplh %v27, %v25
+; vmlf %v29, %v26, %v27
+; vgmf %v31, 17, 17
+; vaf %v1, %v29, %v31
+; vesraf %v3, %v1, 15
+; vpksf %v24, %v23, %v3
+; br %r14
+
+function %sqmul_round_sat(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = sqmul_round_sat.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vuphf %v5, %v24
+; vuphf %v7, %v25
+; lgdr %r3, %f5
+; lgdr %r5, %f7
+; msgr %r3, %r5
+; vlgvg %r5, %v5, 1
+; vlgvg %r4, %v7, 1
+; msgr %r5, %r4
+; vlvgp %v29, %r3, %r5
+; vgmg %v31, 33, 33
+; vag %v1, %v29, %v31
+; vesrag %v3, %v1, 31
+; vuplf %v5, %v24
+; vuplf %v7, %v25
+; lgdr %r3, %f5
+; lgdr %r5, %f7
+; msgr %r3, %r5
+; vlgvg %r5, %v5, 1
+; vlgvg %r4, %v7, 1
+; msgr %r5, %r4
+; vlvgp %v29, %r3, %r5
+; vgmg %v31, 33, 33
+; vag %v1, %v29, %v31
+; vesrag %v4, %v1, 31
+; vpksg %v24, %v3, %v4
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-bitops.clif b/cranelift/filetests/filetests/isa/s390x/vec-bitops.clif
new file mode 100644
index 000000000000..a5cff95c475c
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-bitops.clif
@@ -0,0 +1,43 @@
+test compile precise-output
+target s390x
+
+function %popcnt_i64x2(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = popcnt.i64x2 v0
+ return v1
+}
+
+; block0:
+; vpopctg %v24, %v24
+; br %r14
+
+function %popcnt_i32x4(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = popcnt.i32x4 v0
+ return v1
+}
+
+; block0:
+; vpopctf %v24, %v24
+; br %r14
+
+function %popcnt_i16x8(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = popcnt.i16x8 v0
+ return v1
+}
+
+; block0:
+; vpopcth %v24, %v24
+; br %r14
+
+function %popcnt_i8x16(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = popcnt.i8x16 v0
+ return v1
+}
+
+; block0:
+; vpopctb %v24, %v24
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-bitwise.clif b/cranelift/filetests/filetests/isa/s390x/vec-bitwise.clif
new file mode 100644
index 000000000000..8722a78703b3
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-bitwise.clif
@@ -0,0 +1,364 @@
+
+test compile precise-output
+target s390x
+
+function %band_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = band.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vn %v24, %v24, %v25
+; br %r14
+
+function %band_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = band.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vn %v24, %v24, %v25
+; br %r14
+
+function %band_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = band.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vn %v24, %v24, %v25
+; br %r14
+
+function %band_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = band.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vn %v24, %v24, %v25
+; br %r14
+
+function %bor_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = bor.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vo %v24, %v24, %v25
+; br %r14
+
+function %bor_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = bor.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vo %v24, %v24, %v25
+; br %r14
+
+function %bor_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = bor.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vo %v24, %v24, %v25
+; br %r14
+
+function %bor_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = bor.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vo %v24, %v24, %v25
+; br %r14
+
+function %bxor_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = bxor.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vx %v24, %v24, %v25
+; br %r14
+
+function %bxor_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = bxor.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vx %v24, %v24, %v25
+; br %r14
+
+function %bxor_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = bxor.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vx %v24, %v24, %v25
+; br %r14
+
+function %bxor_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = bxor.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vx %v24, %v24, %v25
+; br %r14
+
+function %band_not_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = band_not.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vnc %v24, %v24, %v25
+; br %r14
+
+function %band_not_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = band_not.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vnc %v24, %v24, %v25
+; br %r14
+
+function %band_not_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = band_not.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vnc %v24, %v24, %v25
+; br %r14
+
+function %band_not_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = band_not.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vnc %v24, %v24, %v25
+; br %r14
+
+function %bor_not_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = bor_not.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; voc %v24, %v24, %v25
+; br %r14
+
+function %bor_not_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = bor_not.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; voc %v24, %v24, %v25
+; br %r14
+
+function %bor_not_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = bor_not.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; voc %v24, %v24, %v25
+; br %r14
+
+function %bor_not_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = bor_not.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; voc %v24, %v24, %v25
+; br %r14
+
+function %bxor_not_i64x2(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = bxor_not.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vnx %v24, %v24, %v25
+; br %r14
+
+function %bxor_not_i32x4(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = bxor_not.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vnx %v24, %v24, %v25
+; br %r14
+
+function %bxor_not_i16x8(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = bxor_not.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vnx %v24, %v24, %v25
+; br %r14
+
+function %bxor_not_i8x16(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = bxor_not.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vnx %v24, %v24, %v25
+; br %r14
+
+function %bnot_i64x2(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = bnot.i64x2 v0
+ return v1
+}
+
+; block0:
+; vno %v24, %v24, %v24
+; br %r14
+
+function %bnot_i32x4(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = bnot.i32x4 v0
+ return v1
+}
+
+; block0:
+; vno %v24, %v24, %v24
+; br %r14
+
+function %bnot_i16x8(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = bnot.i16x8 v0
+ return v1
+}
+
+; block0:
+; vno %v24, %v24, %v24
+; br %r14
+
+function %bnot_i8x16(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = bnot.i8x16 v0
+ return v1
+}
+
+; block0:
+; vno %v24, %v24, %v24
+; br %r14
+
+function %bitselect_i64x2(i64x2, i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2, v2: i64x2):
+ v3 = bitselect.i64x2 v0, v1, v2
+ return v3
+}
+
+; block0:
+; vsel %v24, %v25, %v26, %v24
+; br %r14
+
+function %bitselect_i32x4(i32x4, i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4, v2: i32x4):
+ v3 = bitselect.i32x4 v0, v1, v2
+ return v3
+}
+
+; block0:
+; vsel %v24, %v25, %v26, %v24
+; br %r14
+
+function %bitselect_i16x8(i16x8, i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8, v2: i16x8):
+ v3 = bitselect.i16x8 v0, v1, v2
+ return v3
+}
+
+; block0:
+; vsel %v24, %v25, %v26, %v24
+; br %r14
+
+function %bitselect_i8x16(i8x16, i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16, v2: i8x16):
+ v3 = bitselect.i8x16 v0, v1, v2
+ return v3
+}
+
+; block0:
+; vsel %v24, %v25, %v26, %v24
+; br %r14
+
+function %vselect_i64x2(b64x2, i64x2, i64x2) -> i64x2 {
+block0(v0: b64x2, v1: i64x2, v2: i64x2):
+ v3 = vselect.i64x2 v0, v1, v2
+ return v3
+}
+
+; block0:
+; vsel %v24, %v25, %v26, %v24
+; br %r14
+
+function %vselect_i32x4(b32x4, i32x4, i32x4) -> i32x4 {
+block0(v0: b32x4, v1: i32x4, v2: i32x4):
+ v3 = vselect.i32x4 v0, v1, v2
+ return v3
+}
+
+; block0:
+; vsel %v24, %v25, %v26, %v24
+; br %r14
+
+function %vselect_i16x8(b16x8, i16x8, i16x8) -> i16x8 {
+block0(v0: b16x8, v1: i16x8, v2: i16x8):
+ v3 = vselect.i16x8 v0, v1, v2
+ return v3
+}
+
+; block0:
+; vsel %v24, %v25, %v26, %v24
+; br %r14
+
+function %vselect_i8x16(b8x16, i8x16, i8x16) -> i8x16 {
+block0(v0: b8x16, v1: i8x16, v2: i8x16):
+ v3 = vselect.i8x16 v0, v1, v2
+ return v3
+}
+
+; block0:
+; vsel %v24, %v25, %v26, %v24
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-constants.clif b/cranelift/filetests/filetests/isa/s390x/vec-constants.clif
new file mode 100644
index 000000000000..b5a6969f2b3e
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-constants.clif
@@ -0,0 +1,213 @@
+test compile precise-output
+target s390x
+
+function %vconst_i64x2_zero() -> i64x2 {
+block0:
+ v1 = vconst.i64x2 [0 0]
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; br %r14
+
+function %vconst_i64x2_splat1() -> i64x2 {
+block0:
+ v1 = vconst.i64x2 [32767 32767]
+ return v1
+}
+
+; block0:
+; vrepig %v24, 32767
+; br %r14
+
+function %vconst_i64x2_splat2() -> i64x2 {
+block0:
+ v1 = vconst.i64x2 [-32768 -32768]
+ return v1
+}
+
+; block0:
+; vrepig %v24, -32768
+; br %r14
+
+function %vconst_i64x2_splat3() -> i64x2 {
+block0:
+ v1 = vconst.i64x2 [32768 32768]
+ return v1
+}
+
+; block0:
+; bras %r1, 12 ; data.u64 0x0000000000008000 ; vlrepg %v24, 0(%r1)
+; br %r14
+
+function %vconst_i64x2_splat4() -> i64x2 {
+block0:
+ v1 = vconst.i64x2 [-32769 -32769]
+ return v1
+}
+
+; block0:
+; bras %r1, 12 ; data.u64 0xffffffffffff7fff ; vlrepg %v24, 0(%r1)
+; br %r14
+
+function %vconst_i64x2_mixed() -> i64x2 {
+block0:
+ v1 = vconst.i64x2 [1 2]
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x00000000000000020000000000000001 ; vl %v24, 0(%r1)
+; br %r14
+
+function %vconst_i32x4_zero() -> i32x4 {
+block0:
+ v1 = vconst.i32x4 [0 0 0 0]
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; br %r14
+
+function %vconst_i32x4_splat1() -> i32x4 {
+block0:
+ v1 = vconst.i32x4 [32767 32767 32767 32767]
+ return v1
+}
+
+; block0:
+; vrepif %v24, 32767
+; br %r14
+
+function %vconst_i32x4_splat2() -> i32x4 {
+block0:
+ v1 = vconst.i32x4 [-32768 -32768 -32768 -32768]
+ return v1
+}
+
+; block0:
+; vrepif %v24, -32768
+; br %r14
+
+function %vconst_i32x4_splat3() -> i32x4 {
+block0:
+ v1 = vconst.i32x4 [32768 32768 32768 32768]
+ return v1
+}
+
+; block0:
+; bras %r1, 8 ; data.u32 0x00008000 ; vlrepf %v24, 0(%r1)
+; br %r14
+
+function %vconst_i32x4_splat4() -> i32x4 {
+block0:
+ v1 = vconst.i32x4 [-32769 -32769 -32769 -32769]
+ return v1
+}
+
+; block0:
+; bras %r1, 8 ; data.u32 0xffff7fff ; vlrepf %v24, 0(%r1)
+; br %r14
+
+function %vconst_i32x4_splat_i64() -> i32x4 {
+block0:
+ v1 = vconst.i32x4 [1 2 1 2]
+ return v1
+}
+
+; block0:
+; bras %r1, 12 ; data.u64 0x0000000200000001 ; vlrepg %v24, 0(%r1)
+; br %r14
+
+function %vconst_i32x4_mixed() -> i32x4 {
+block0:
+ v1 = vconst.i32x4 [1 2 3 4]
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x00000004000000030000000200000001 ; vl %v24, 0(%r1)
+; br %r14
+
+function %vconst_i16x8_zero() -> i16x8 {
+block0:
+ v1 = vconst.i16x8 [0 0 0 0 0 0 0 0]
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; br %r14
+
+function %vconst_i16x8_splat1() -> i16x8 {
+block0:
+ v1 = vconst.i16x8 [32767 32767 32767 32767 32767 32767 32767 32767]
+ return v1
+}
+
+; block0:
+; vrepih %v24, 32767
+; br %r14
+
+function %vconst_i16x8_splat2() -> i16x8 {
+block0:
+ v1 = vconst.i16x8 [-32768 -32768 -32768 -32768 -32768 -32768 -32768 -32768]
+ return v1
+}
+
+; block0:
+; vrepih %v24, -32768
+; br %r14
+
+function %vconst_i16x8_mixed() -> i16x8 {
+block0:
+ v1 = vconst.i16x8 [1 2 3 4 5 6 7 8]
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x00080007000600050004000300020001 ; vl %v24, 0(%r1)
+; br %r14
+
+function %vconst_i8x16_zero() -> i8x16 {
+block0:
+ v1 = vconst.i8x16 [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; br %r14
+
+function %vconst_i8x16_splat1() -> i8x16 {
+block0:
+ v1 = vconst.i8x16 [127 127 127 127 127 127 127 127 127 127 127 127 127 127 127 127]
+ return v1
+}
+
+; block0:
+; vrepib %v24, 127
+; br %r14
+
+function %vconst_i8x16_splat2() -> i8x16 {
+block0:
+ v1 = vconst.i8x16 [-128 -128 -128 -128 -128 -128 -128 -128 -128 -128 -128 -128 -128 -128 -128 -128]
+ return v1
+}
+
+; block0:
+; vrepib %v24, 128
+; br %r14
+
+function %vconst_i8x16_mixed() -> i8x16 {
+block0:
+ v1 = vconst.i8x16 [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16]
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x100f0e0d0c0b0a090807060504030201 ; vl %v24, 0(%r1)
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-conversions.clif b/cranelift/filetests/filetests/isa/s390x/vec-conversions.clif
new file mode 100644
index 000000000000..b137c8cd214b
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-conversions.clif
@@ -0,0 +1,222 @@
+test compile precise-output
+target s390x
+
+function %snarrow_i64x2_i32x4(i64x2, i64x2) -> i32x4 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = snarrow.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vpksg %v24, %v25, %v24
+; br %r14
+
+function %snarrow_i32x4_i16x8(i32x4, i32x4) -> i16x8 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = snarrow.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vpksf %v24, %v25, %v24
+; br %r14
+
+function %snarrow_i16x8_i8x16(i16x8, i16x8) -> i8x16 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = snarrow.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vpksh %v24, %v25, %v24
+; br %r14
+
+function %unarrow_i64x2_i32x4(i64x2, i64x2) -> i32x4 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = unarrow.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vgbm %v5, 0
+; vmxg %v7, %v25, %v5
+; vmxg %v17, %v24, %v5
+; vpklsg %v24, %v7, %v17
+; br %r14
+
+function %unarrow_i32x4_i16x8(i32x4, i32x4) -> i16x8 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = unarrow.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vgbm %v5, 0
+; vmxf %v7, %v25, %v5
+; vmxf %v17, %v24, %v5
+; vpklsf %v24, %v7, %v17
+; br %r14
+
+function %unarrow_i16x8_i8x16(i16x8, i16x8) -> i8x16 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = unarrow.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vgbm %v5, 0
+; vmxh %v7, %v25, %v5
+; vmxh %v17, %v24, %v5
+; vpklsh %v24, %v7, %v17
+; br %r14
+
+function %uunarrow_i64x2_i32x4(i64x2, i64x2) -> i32x4 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = uunarrow.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vpklsg %v24, %v25, %v24
+; br %r14
+
+function %uunarrow_i32x4_i16x8(i32x4, i32x4) -> i16x8 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = uunarrow.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vpklsf %v24, %v25, %v24
+; br %r14
+
+function %uunarrow_i16x8_i8x16(i16x8, i16x8) -> i8x16 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = uunarrow.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vpklsh %v24, %v25, %v24
+; br %r14
+
+function %swiden_low_i32x4_i64x2(i32x4) -> i64x2 {
+block0(v0: i32x4):
+ v1 = swiden_low.i32x4 v0
+ return v1
+}
+
+; block0:
+; vuplf %v24, %v24
+; br %r14
+
+function %swiden_low_i16x8_i32x4(i16x8) -> i32x4 {
+block0(v0: i16x8):
+ v1 = swiden_low.i16x8 v0
+ return v1
+}
+
+; block0:
+; vuplh %v24, %v24
+; br %r14
+
+function %swiden_low_i8x16_i16x8(i8x16) -> i16x8 {
+block0(v0: i8x16):
+ v1 = swiden_low.i8x16 v0
+ return v1
+}
+
+; block0:
+; vuplb %v24, %v24
+; br %r14
+
+function %swiden_high_i32x4_i64x2(i32x4) -> i64x2 {
+block0(v0: i32x4):
+ v1 = swiden_high.i32x4 v0
+ return v1
+}
+
+; block0:
+; vuphf %v24, %v24
+; br %r14
+
+function %swiden_high_i16x8_i32x4(i16x8) -> i32x4 {
+block0(v0: i16x8):
+ v1 = swiden_high.i16x8 v0
+ return v1
+}
+
+; block0:
+; vuphh %v24, %v24
+; br %r14
+
+function %swiden_high_i8x16_i16x8(i8x16) -> i16x8 {
+block0(v0: i8x16):
+ v1 = swiden_high.i8x16 v0
+ return v1
+}
+
+; block0:
+; vuphb %v24, %v24
+; br %r14
+
+function %uwiden_low_i32x4_i64x2(i32x4) -> i64x2 {
+block0(v0: i32x4):
+ v1 = uwiden_low.i32x4 v0
+ return v1
+}
+
+; block0:
+; vupllf %v24, %v24
+; br %r14
+
+function %uwiden_low_i16x8_i32x4(i16x8) -> i32x4 {
+block0(v0: i16x8):
+ v1 = uwiden_low.i16x8 v0
+ return v1
+}
+
+; block0:
+; vupllh %v24, %v24
+; br %r14
+
+function %uwiden_low_i8x16_i16x8(i8x16) -> i16x8 {
+block0(v0: i8x16):
+ v1 = uwiden_low.i8x16 v0
+ return v1
+}
+
+; block0:
+; vupllb %v24, %v24
+; br %r14
+
+function %uwiden_high_i32x4_i64x2(i32x4) -> i64x2 {
+block0(v0: i32x4):
+ v1 = uwiden_high.i32x4 v0
+ return v1
+}
+
+; block0:
+; vuplhf %v24, %v24
+; br %r14
+
+function %uwiden_high_i16x8_i32x4(i16x8) -> i32x4 {
+block0(v0: i16x8):
+ v1 = uwiden_high.i16x8 v0
+ return v1
+}
+
+; block0:
+; vuplhh %v24, %v24
+; br %r14
+
+function %uwiden_high_i8x16_i16x8(i8x16) -> i16x8 {
+block0(v0: i8x16):
+ v1 = uwiden_high.i8x16 v0
+ return v1
+}
+
+; block0:
+; vuplhb %v24, %v24
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-fcmp.clif b/cranelift/filetests/filetests/isa/s390x/vec-fcmp.clif
new file mode 100644
index 000000000000..32aeab3bd15d
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-fcmp.clif
@@ -0,0 +1,309 @@
+test compile precise-output
+target s390x
+
+function %fcmp_eq_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 eq v0, v1
+ return v2
+}
+
+; block0:
+; vfcedb %v24, %v24, %v25
+; br %r14
+
+function %fcmp_ne_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 ne v0, v1
+ return v2
+}
+
+; block0:
+; vfcedb %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_gt_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 gt v0, v1
+ return v2
+}
+
+; block0:
+; vfchdb %v24, %v24, %v25
+; br %r14
+
+function %fcmp_lt_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 lt v0, v1
+ return v2
+}
+
+; block0:
+; vfchdb %v24, %v25, %v24
+; br %r14
+
+function %fcmp_ge_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 ge v0, v1
+ return v2
+}
+
+; block0:
+; vfchedb %v24, %v24, %v25
+; br %r14
+
+function %fcmp_le_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 le v0, v1
+ return v2
+}
+
+; block0:
+; vfchedb %v24, %v25, %v24
+; br %r14
+
+function %fcmp_ueq_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 ueq v0, v1
+ return v2
+}
+
+; block0:
+; vfchdb %v5, %v24, %v25
+; vfchdb %v7, %v25, %v24
+; vno %v24, %v5, %v7
+; br %r14
+
+function %fcmp_one_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 one v0, v1
+ return v2
+}
+
+; block0:
+; vfchdb %v5, %v24, %v25
+; vfchdb %v7, %v25, %v24
+; vo %v24, %v5, %v7
+; br %r14
+
+function %fcmp_ugt_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 ugt v0, v1
+ return v2
+}
+
+; block0:
+; vfchedb %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_ult_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 ult v0, v1
+ return v2
+}
+
+; block0:
+; vfchedb %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_uge_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 uge v0, v1
+ return v2
+}
+
+; block0:
+; vfchdb %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_ule_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 ule v0, v1
+ return v2
+}
+
+; block0:
+; vfchdb %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_ord_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 ord v0, v1
+ return v2
+}
+
+; block0:
+; vfchedb %v5, %v24, %v25
+; vfchedb %v7, %v25, %v24
+; vo %v24, %v5, %v7
+; br %r14
+
+function %fcmp_uno_f64x2(f64x2, f64x2) -> b64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp.f64x2 uno v0, v1
+ return v2
+}
+
+; block0:
+; vfchedb %v5, %v24, %v25
+; vfchedb %v7, %v25, %v24
+; vno %v24, %v5, %v7
+; br %r14
+
+function %fcmp_eq_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 eq v0, v1
+ return v2
+}
+
+; block0:
+; vfcesb %v24, %v24, %v25
+; br %r14
+
+function %fcmp_ne_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 ne v0, v1
+ return v2
+}
+
+; block0:
+; vfcesb %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_gt_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 gt v0, v1
+ return v2
+}
+
+; block0:
+; vfchsb %v24, %v24, %v25
+; br %r14
+
+function %fcmp_lt_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 lt v0, v1
+ return v2
+}
+
+; block0:
+; vfchsb %v24, %v25, %v24
+; br %r14
+
+function %fcmp_ge_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 ge v0, v1
+ return v2
+}
+
+; block0:
+; vfchesb %v24, %v24, %v25
+; br %r14
+
+function %fcmp_le_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 le v0, v1
+ return v2
+}
+
+; block0:
+; vfchesb %v24, %v25, %v24
+; br %r14
+
+function %fcmp_ueq_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 ueq v0, v1
+ return v2
+}
+
+; block0:
+; vfchsb %v5, %v24, %v25
+; vfchsb %v7, %v25, %v24
+; vno %v24, %v5, %v7
+; br %r14
+
+function %fcmp_one_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 one v0, v1
+ return v2
+}
+
+; block0:
+; vfchsb %v5, %v24, %v25
+; vfchsb %v7, %v25, %v24
+; vo %v24, %v5, %v7
+; br %r14
+
+function %fcmp_ugt_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 ugt v0, v1
+ return v2
+}
+
+; block0:
+; vfchesb %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_ult_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 ult v0, v1
+ return v2
+}
+
+; block0:
+; vfchesb %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_uge_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 uge v0, v1
+ return v2
+}
+
+; block0:
+; vfchsb %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_ule_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 ule v0, v1
+ return v2
+}
+
+; block0:
+; vfchsb %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %fcmp_ord_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 ord v0, v1
+ return v2
+}
+
+; block0:
+; vfchesb %v5, %v24, %v25
+; vfchesb %v7, %v25, %v24
+; vo %v24, %v5, %v7
+; br %r14
+
+function %fcmp_uno_f32x4(f32x4, f32x4) -> b32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcmp.f32x4 uno v0, v1
+ return v2
+}
+
+; block0:
+; vfchesb %v5, %v24, %v25
+; vfchesb %v7, %v25, %v24
+; vno %v24, %v5, %v7
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-fp-arch13.clif b/cranelift/filetests/filetests/isa/s390x/vec-fp-arch13.clif
new file mode 100644
index 000000000000..4c00c348d458
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-fp-arch13.clif
@@ -0,0 +1,90 @@
+test compile precise-output
+target s390x arch13
+
+function %fcvt_from_uint_i32x4_f32x4(i32x4) -> f32x4 {
+block0(v0: i32x4):
+ v1 = fcvt_from_uint.f32x4 v0
+ return v1
+}
+
+; block0:
+; vcelfb %v24, %v24, 0, 4
+; br %r14
+
+function %fcvt_from_sint_i32x4_f32x4(i32x4) -> f32x4 {
+block0(v0: i32x4):
+ v1 = fcvt_from_sint.f32x4 v0
+ return v1
+}
+
+; block0:
+; vcefb %v24, %v24, 0, 4
+; br %r14
+
+function %fcvt_from_uint_i64x2_f64x2(i64x2) -> f64x2 {
+block0(v0: i64x2):
+ v1 = fcvt_from_uint.f64x2 v0
+ return v1
+}
+
+; block0:
+; vcdlgb %v24, %v24, 0, 4
+; br %r14
+
+function %fcvt_from_sint_i64x2_f64x2(i64x2) -> f64x2 {
+block0(v0: i64x2):
+ v1 = fcvt_from_sint.f64x2 v0
+ return v1
+}
+
+; block0:
+; vcdgb %v24, %v24, 0, 4
+; br %r14
+
+
+function %fcvt_to_uint_sat_f32x4_i32x4(f32x4) -> i32x4 {
+block0(v0: f32x4):
+ v1 = fcvt_to_uint_sat.i32x4 v0
+ return v1
+}
+
+; block0:
+; vclfeb %v24, %v24, 0, 5
+; br %r14
+
+function %fcvt_to_sint_sat_f32x4_i32x4(f32x4) -> i32x4 {
+block0(v0: f32x4):
+ v1 = fcvt_to_sint_sat.i32x4 v0
+ return v1
+}
+
+; block0:
+; vcfeb %v3, %v24, 0, 5
+; vgbm %v5, 0
+; vfcesb %v7, %v24, %v24
+; vsel %v24, %v3, %v5, %v7
+; br %r14
+
+function %fcvt_to_uint_sat_f64x2_i64x2(f64x2) -> i64x2 {
+block0(v0: f64x2):
+ v1 = fcvt_to_uint_sat.i64x2 v0
+ return v1
+}
+
+; block0:
+; vclgdb %v24, %v24, 0, 5
+; br %r14
+
+function %fcvt_to_sint_sat_f64x2_i64x2(f64x2) -> i64x2 {
+block0(v0: f64x2):
+ v1 = fcvt_to_sint_sat.i64x2 v0
+ return v1
+}
+
+; block0:
+; vcgdb %v3, %v24, 0, 5
+; vgbm %v5, 0
+; vfcedb %v7, %v24, %v24
+; vsel %v24, %v3, %v5, %v7
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-fp.clif b/cranelift/filetests/filetests/isa/s390x/vec-fp.clif
new file mode 100644
index 000000000000..fc356d57a762
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-fp.clif
@@ -0,0 +1,533 @@
+test compile precise-output
+target s390x
+
+function %vconst_f32x4_zero() -> f32x4 {
+block0:
+ v1 = vconst.f32x4 [0x0.0 0x0.0 0x0.0 0x0.0]
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; br %r14
+
+function %vconst_f64x2_zero() -> f64x2 {
+block0:
+ v1 = vconst.f64x2 [0x0.0 0x0.0]
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; br %r14
+
+function %vconst_f32x4_mixed() -> f32x4 {
+block0:
+ v1 = vconst.f32x4 [0x1.0 0x2.0 0x3.0 0x4.0]
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x4080000040400000400000003f800000 ; vl %v24, 0(%r1)
+; br %r14
+
+function %vconst_f64x2_mixed() -> f64x2 {
+block0:
+ v1 = vconst.f64x2 [0x1.0 0x2.0]
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x40000000000000003ff0000000000000 ; vl %v24, 0(%r1)
+; br %r14
+
+function %fadd_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fadd v0, v1
+ return v2
+}
+
+; block0:
+; vfasb %v24, %v24, %v25
+; br %r14
+
+function %fadd_f64x2(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fadd v0, v1
+ return v2
+}
+
+; block0:
+; vfadb %v24, %v24, %v25
+; br %r14
+
+function %fsub_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fsub v0, v1
+ return v2
+}
+
+; block0:
+; vfssb %v24, %v24, %v25
+; br %r14
+
+function %fsub_f64x2(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fsub v0, v1
+ return v2
+}
+
+; block0:
+; vfsdb %v24, %v24, %v25
+; br %r14
+
+function %fmul_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fmul v0, v1
+ return v2
+}
+
+; block0:
+; vfmsb %v24, %v24, %v25
+; br %r14
+
+function %fmul_f64x2(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fmul v0, v1
+ return v2
+}
+
+; block0:
+; vfmdb %v24, %v24, %v25
+; br %r14
+
+function %fdiv_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fdiv v0, v1
+ return v2
+}
+
+; block0:
+; vfdsb %v24, %v24, %v25
+; br %r14
+
+function %fdiv_f64x2(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fdiv v0, v1
+ return v2
+}
+
+; block0:
+; vfddb %v24, %v24, %v25
+; br %r14
+
+function %fmin_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fmin v0, v1
+ return v2
+}
+
+; block0:
+; vfminsb %v24, %v24, %v25, 1
+; br %r14
+
+function %fmin_f64x2(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fmin v0, v1
+ return v2
+}
+
+; block0:
+; vfmindb %v24, %v24, %v25, 1
+; br %r14
+
+function %fmax_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fmax v0, v1
+ return v2
+}
+
+; block0:
+; vfmaxsb %v24, %v24, %v25, 1
+; br %r14
+
+function %fmax_f64x2(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fmax v0, v1
+ return v2
+}
+
+; block0:
+; vfmaxdb %v24, %v24, %v25, 1
+; br %r14
+
+function %fmin_pseudo_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fmin_pseudo v0, v1
+ return v2
+}
+
+; block0:
+; vfminsb %v24, %v24, %v25, 3
+; br %r14
+
+function %fmin_pseudo_f64x2(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fmin_pseudo v0, v1
+ return v2
+}
+
+; block0:
+; vfmindb %v24, %v24, %v25, 3
+; br %r14
+
+function %fmax_pseudo_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fmax_pseudo v0, v1
+ return v2
+}
+
+; block0:
+; vfmaxsb %v24, %v24, %v25, 3
+; br %r14
+
+function %fmax_pseudo_f64x2(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fmax_pseudo v0, v1
+ return v2
+}
+
+; block0:
+; vfmaxdb %v24, %v24, %v25, 3
+; br %r14
+
+function %sqrt_f32x4(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = sqrt v0
+ return v1
+}
+
+; block0:
+; vfsqsb %v24, %v24
+; br %r14
+
+function %sqrt_f64x2(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = sqrt v0
+ return v1
+}
+
+; block0:
+; vfsqdb %v24, %v24
+; br %r14
+
+function %fabs_f32x4(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = fabs v0
+ return v1
+}
+
+; block0:
+; vflpsb %v24, %v24
+; br %r14
+
+function %fabs_f64x2(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = fabs v0
+ return v1
+}
+
+; block0:
+; vflpdb %v24, %v24
+; br %r14
+
+function %fneg_f32x4(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = fneg v0
+ return v1
+}
+
+; block0:
+; vflcsb %v24, %v24
+; br %r14
+
+function %fneg_f64x2(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = fneg v0
+ return v1
+}
+
+; block0:
+; vflcdb %v24, %v24
+; br %r14
+
+function %fvpromote_low_f32x4(f32x4) -> f64x2 {
+block0(v0: f32x4):
+ v1 = fvpromote_low v0
+ return v1
+}
+
+; block0:
+; vmrlf %v3, %v24, %v24
+; vldeb %v24, %v3
+; br %r14
+
+function %fvdemote_f64x2(f64x2) -> f32x4 {
+block0(v0: f64x2):
+ v1 = fvdemote v0
+ return v1
+}
+
+; block0:
+; vledb %v3, %v24, 0, 0
+; vgbm %v5, 0
+; bras %r1, 20 ; data.u128 0x10101010101010100001020308090a0b ; vl %v7, 0(%r1)
+; vperm %v24, %v3, %v5, %v7
+; br %r14
+
+function %ceil_f32x4(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = ceil v0
+ return v1
+}
+
+; block0:
+; vfisb %v24, %v24, 0, 6
+; br %r14
+
+function %ceil_f64x2(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = ceil v0
+ return v1
+}
+
+; block0:
+; vfidb %v24, %v24, 0, 6
+; br %r14
+
+function %floor_f32x4(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = floor v0
+ return v1
+}
+
+; block0:
+; vfisb %v24, %v24, 0, 7
+; br %r14
+
+function %floor_f64x2(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = floor v0
+ return v1
+}
+
+; block0:
+; vfidb %v24, %v24, 0, 7
+; br %r14
+
+function %trunc_f32x4(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = trunc v0
+ return v1
+}
+
+; block0:
+; vfisb %v24, %v24, 0, 5
+; br %r14
+
+function %trunc_f64x2(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = trunc v0
+ return v1
+}
+
+; block0:
+; vfidb %v24, %v24, 0, 5
+; br %r14
+
+function %nearest_f32x4(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = nearest v0
+ return v1
+}
+
+; block0:
+; vfisb %v24, %v24, 0, 4
+; br %r14
+
+function %nearest_f64x2(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = nearest v0
+ return v1
+}
+
+; block0:
+; vfidb %v24, %v24, 0, 4
+; br %r14
+
+function %fma_f32x4(f32x4, f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4, v2: f32x4):
+ v3 = fma v0, v1, v2
+ return v3
+}
+
+; block0:
+; vfmasb %v24, %v24, %v25, %v26
+; br %r14
+
+function %fma_f64x2(f64x2, f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2, v2: f64x2):
+ v3 = fma v0, v1, v2
+ return v3
+}
+
+; block0:
+; vfmadb %v24, %v24, %v25, %v26
+; br %r14
+
+function %fcopysign_f32x4(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = fcopysign v0, v1
+ return v2
+}
+
+; block0:
+; vgmf %v5, 1, 31
+; vsel %v24, %v24, %v25, %v5
+; br %r14
+
+function %fcopysign_f64x2(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcopysign v0, v1
+ return v2
+}
+
+; block0:
+; vgmg %v5, 1, 63
+; vsel %v24, %v24, %v25, %v5
+; br %r14
+
+function %fcvt_from_uint_i32x4_f32x4(i32x4) -> f32x4 {
+block0(v0: i32x4):
+ v1 = fcvt_from_uint.f32x4 v0
+ return v1
+}
+
+; block0:
+; vuplhf %v3, %v24
+; vcdlgb %v5, %v3, 0, 3
+; vledb %v7, %v5, 0, 4
+; vupllf %v17, %v24
+; vcdlgb %v19, %v17, 0, 3
+; vledb %v21, %v19, 0, 4
+; bras %r1, 20 ; data.u128 0x0001020308090a0b1011121318191a1b ; vl %v23, 0(%r1)
+; vperm %v24, %v7, %v21, %v23
+; br %r14
+
+function %fcvt_from_sint_i32x4_f32x4(i32x4) -> f32x4 {
+block0(v0: i32x4):
+ v1 = fcvt_from_sint.f32x4 v0
+ return v1
+}
+
+; block0:
+; vuphf %v3, %v24
+; vcdgb %v5, %v3, 0, 3
+; vledb %v7, %v5, 0, 4
+; vuplf %v17, %v24
+; vcdgb %v19, %v17, 0, 3
+; vledb %v21, %v19, 0, 4
+; bras %r1, 20 ; data.u128 0x0001020308090a0b1011121318191a1b ; vl %v23, 0(%r1)
+; vperm %v24, %v7, %v21, %v23
+; br %r14
+
+function %fcvt_from_uint_i64x2_f64x2(i64x2) -> f64x2 {
+block0(v0: i64x2):
+ v1 = fcvt_from_uint.f64x2 v0
+ return v1
+}
+
+; block0:
+; vcdlgb %v24, %v24, 0, 4
+; br %r14
+
+function %fcvt_from_sint_i64x2_f64x2(i64x2) -> f64x2 {
+block0(v0: i64x2):
+ v1 = fcvt_from_sint.f64x2 v0
+ return v1
+}
+
+; block0:
+; vcdgb %v24, %v24, 0, 4
+; br %r14
+
+
+function %fcvt_low_from_sint_i32x4_f64x2(i32x4) -> f64x2 {
+block0(v0: i32x4):
+ v1 = fcvt_low_from_sint.f64x2 v0
+ return v1
+}
+
+; block0:
+; vuplf %v3, %v24
+; vcdgb %v24, %v3, 0, 4
+; br %r14
+
+function %fcvt_to_uint_sat_f32x4_i32x4(f32x4) -> i32x4 {
+block0(v0: f32x4):
+ v1 = fcvt_to_uint_sat.i32x4 v0
+ return v1
+}
+
+; block0:
+; vmrhf %v3, %v24, %v24
+; vldeb %v5, %v3
+; vclgdb %v7, %v5, 0, 5
+; vmrlf %v17, %v24, %v24
+; vldeb %v19, %v17
+; vclgdb %v21, %v19, 0, 5
+; vpklsg %v24, %v7, %v21
+; br %r14
+
+function %fcvt_to_sint_sat_f32x4_i32x4(f32x4) -> i32x4 {
+block0(v0: f32x4):
+ v1 = fcvt_to_sint_sat.i32x4 v0
+ return v1
+}
+
+; block0:
+; vmrhf %v3, %v24, %v24
+; vldeb %v5, %v3
+; vcgdb %v7, %v5, 0, 5
+; vmrlf %v17, %v24, %v24
+; vldeb %v19, %v17
+; vcgdb %v21, %v19, 0, 5
+; vpksg %v23, %v7, %v21
+; vgbm %v25, 0
+; vfcesb %v27, %v24, %v24
+; vsel %v24, %v23, %v25, %v27
+; br %r14
+
+function %fcvt_to_uint_sat_f64x2_i64x2(f64x2) -> i64x2 {
+block0(v0: f64x2):
+ v1 = fcvt_to_uint_sat.i64x2 v0
+ return v1
+}
+
+; block0:
+; vclgdb %v24, %v24, 0, 5
+; br %r14
+
+function %fcvt_to_sint_sat_f64x2_i64x2(f64x2) -> i64x2 {
+block0(v0: f64x2):
+ v1 = fcvt_to_sint_sat.i64x2 v0
+ return v1
+}
+
+; block0:
+; vcgdb %v3, %v24, 0, 5
+; vgbm %v5, 0
+; vfcedb %v7, %v24, %v24
+; vsel %v24, %v3, %v5, %v7
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-icmp.clif b/cranelift/filetests/filetests/isa/s390x/vec-icmp.clif
new file mode 100644
index 000000000000..fe9e6fead830
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-icmp.clif
@@ -0,0 +1,423 @@
+test compile precise-output
+target s390x
+
+function %icmp_eq_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 eq v0, v1
+ return v2
+}
+
+; block0:
+; vceqg %v24, %v24, %v25
+; br %r14
+
+function %icmp_ne_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 ne v0, v1
+ return v2
+}
+
+; block0:
+; vceqg %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_sgt_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 sgt v0, v1
+ return v2
+}
+
+; block0:
+; vchg %v24, %v24, %v25
+; br %r14
+
+function %icmp_slt_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 slt v0, v1
+ return v2
+}
+
+; block0:
+; vchg %v24, %v25, %v24
+; br %r14
+
+function %icmp_sge_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 sge v0, v1
+ return v2
+}
+
+; block0:
+; vchg %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_sle_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 sle v0, v1
+ return v2
+}
+
+; block0:
+; vchg %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_ugt_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 ugt v0, v1
+ return v2
+}
+
+; block0:
+; vchlg %v24, %v24, %v25
+; br %r14
+
+function %icmp_ult_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 ult v0, v1
+ return v2
+}
+
+; block0:
+; vchlg %v24, %v25, %v24
+; br %r14
+
+function %icmp_uge_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 uge v0, v1
+ return v2
+}
+
+; block0:
+; vchlg %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_ule_i64x2(i64x2, i64x2) -> b64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp.i64x2 ule v0, v1
+ return v2
+}
+
+; block0:
+; vchlg %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_eq_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 eq v0, v1
+ return v2
+}
+
+; block0:
+; vceqf %v24, %v24, %v25
+; br %r14
+
+function %icmp_ne_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 ne v0, v1
+ return v2
+}
+
+; block0:
+; vceqf %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_sgt_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 sgt v0, v1
+ return v2
+}
+
+; block0:
+; vchf %v24, %v24, %v25
+; br %r14
+
+function %icmp_slt_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 slt v0, v1
+ return v2
+}
+
+; block0:
+; vchf %v24, %v25, %v24
+; br %r14
+
+function %icmp_sge_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 sge v0, v1
+ return v2
+}
+
+; block0:
+; vchf %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_sle_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 sle v0, v1
+ return v2
+}
+
+; block0:
+; vchf %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_ugt_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 ugt v0, v1
+ return v2
+}
+
+; block0:
+; vchlf %v24, %v24, %v25
+; br %r14
+
+function %icmp_ult_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 ult v0, v1
+ return v2
+}
+
+; block0:
+; vchlf %v24, %v25, %v24
+; br %r14
+
+function %icmp_uge_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 uge v0, v1
+ return v2
+}
+
+; block0:
+; vchlf %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_ule_i32x4(i32x4, i32x4) -> b32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = icmp.i32x4 ule v0, v1
+ return v2
+}
+
+; block0:
+; vchlf %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_eq_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 eq v0, v1
+ return v2
+}
+
+; block0:
+; vceqh %v24, %v24, %v25
+; br %r14
+
+function %icmp_ne_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 ne v0, v1
+ return v2
+}
+
+; block0:
+; vceqh %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_sgt_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 sgt v0, v1
+ return v2
+}
+
+; block0:
+; vchh %v24, %v24, %v25
+; br %r14
+
+function %icmp_slt_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 slt v0, v1
+ return v2
+}
+
+; block0:
+; vchh %v24, %v25, %v24
+; br %r14
+
+function %icmp_sge_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 sge v0, v1
+ return v2
+}
+
+; block0:
+; vchh %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_sle_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 sle v0, v1
+ return v2
+}
+
+; block0:
+; vchh %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_ugt_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 ugt v0, v1
+ return v2
+}
+
+; block0:
+; vchlh %v24, %v24, %v25
+; br %r14
+
+function %icmp_ult_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 ult v0, v1
+ return v2
+}
+
+; block0:
+; vchlh %v24, %v25, %v24
+; br %r14
+
+function %icmp_uge_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 uge v0, v1
+ return v2
+}
+
+; block0:
+; vchlh %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_ule_i16x8(i16x8, i16x8) -> b16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = icmp.i16x8 ule v0, v1
+ return v2
+}
+
+; block0:
+; vchlh %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_eq_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 eq v0, v1
+ return v2
+}
+
+; block0:
+; vceqb %v24, %v24, %v25
+; br %r14
+
+function %icmp_ne_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 ne v0, v1
+ return v2
+}
+
+; block0:
+; vceqb %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_sgt_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 sgt v0, v1
+ return v2
+}
+
+; block0:
+; vchb %v24, %v24, %v25
+; br %r14
+
+function %icmp_slt_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 slt v0, v1
+ return v2
+}
+
+; block0:
+; vchb %v24, %v25, %v24
+; br %r14
+
+function %icmp_sge_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 sge v0, v1
+ return v2
+}
+
+; block0:
+; vchb %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_sle_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 sle v0, v1
+ return v2
+}
+
+; block0:
+; vchb %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_ugt_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 ugt v0, v1
+ return v2
+}
+
+; block0:
+; vchlb %v24, %v24, %v25
+; br %r14
+
+function %icmp_ult_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 ult v0, v1
+ return v2
+}
+
+; block0:
+; vchlb %v24, %v25, %v24
+; br %r14
+
+function %icmp_uge_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 uge v0, v1
+ return v2
+}
+
+; block0:
+; vchlb %v5, %v25, %v24
+; vno %v24, %v5, %v5
+; br %r14
+
+function %icmp_ule_i8x16(i8x16, i8x16) -> b8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = icmp.i8x16 ule v0, v1
+ return v2
+}
+
+; block0:
+; vchlb %v5, %v24, %v25
+; vno %v24, %v5, %v5
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-lane-arch13.clif b/cranelift/filetests/filetests/isa/s390x/vec-lane-arch13.clif
new file mode 100644
index 000000000000..5ee1ef906fa6
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-lane-arch13.clif
@@ -0,0 +1,807 @@
+test compile precise-output
+target s390x arch13
+
+function %insertlane_i64x2_mem_0(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = load.i64 v1
+ v3 = insertlane.i64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleg %v24, 0(%r2), 1
+; br %r14
+
+function %insertlane_i64x2_mem_1(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = load.i64 v1
+ v3 = insertlane.i64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vleg %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i64x2_mem_little_0(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = load.i64 little v1
+ v3 = insertlane.i64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vlebrg %v24, 0(%r2), 1
+; br %r14
+
+function %insertlane_i64x2_mem_little_1(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = load.i64 little v1
+ v3 = insertlane.i64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vlebrg %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i32x4_mem_0(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 v1
+ v3 = insertlane.i32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vlef %v24, 0(%r2), 3
+; br %r14
+
+function %insertlane_i32x4_mem_3(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 v1
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vlef %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i32x4_mem_little_0(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 little v1
+ v3 = insertlane.i32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vlebrf %v24, 0(%r2), 3
+; br %r14
+
+function %insertlane_i32x4_mem_little_3(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 little v1
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vlebrf %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i16x8_mem_0(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+ v2 = load.i16 v1
+ v3 = insertlane.i16x8 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleh %v24, 0(%r2), 7
+; br %r14
+
+function %insertlane_i16x8_mem_7(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+ v2 = load.i16 v1
+ v3 = insertlane.i16x8 v0, v2, 7
+ return v3
+}
+
+; block0:
+; vleh %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i16x8_mem_little_0(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+ v2 = load.i16 little v1
+ v3 = insertlane.i16x8 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vlebrh %v24, 0(%r2), 7
+; br %r14
+
+function %insertlane_i16x8_mem_little_7(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+ v2 = load.i16 little v1
+ v3 = insertlane.i16x8 v0, v2, 7
+ return v3
+}
+
+; block0:
+; vlebrh %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i8x16_mem_0(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+ v2 = load.i8 v1
+ v3 = insertlane.i8x16 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleb %v24, 0(%r2), 15
+; br %r14
+
+function %insertlane_i8x16_mem_15(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+ v2 = load.i8 v1
+ v3 = insertlane.i8x16 v0, v2, 15
+ return v3
+}
+
+; block0:
+; vleb %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i8x16_mem_little_0(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+ v2 = load.i8 little v1
+ v3 = insertlane.i8x16 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleb %v24, 0(%r2), 15
+; br %r14
+
+function %insertlane_i8x16_mem_little_15(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+ v2 = load.i8 little v1
+ v3 = insertlane.i8x16 v0, v2, 15
+ return v3
+}
+
+; block0:
+; vleb %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_f64x2_mem_0(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+ v2 = load.f64 v1
+ v3 = insertlane.f64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleg %v24, 0(%r2), 1
+; br %r14
+
+function %insertlane_f64x2_mem_1(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+ v2 = load.f64 v1
+ v3 = insertlane.f64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vleg %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_f64x2_mem_little_0(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+ v2 = load.f64 little v1
+ v3 = insertlane.f64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vlebrg %v24, 0(%r2), 1
+; br %r14
+
+function %insertlane_f64x2_mem_little_1(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+ v2 = load.f64 little v1
+ v3 = insertlane.f64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vlebrg %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_f32x4_mem_0(f32x4, i64) -> f32x4 {
+block0(v0: f32x4, v1: i64):
+ v2 = load.f32 v1
+ v3 = insertlane.f32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vlef %v24, 0(%r2), 3
+; br %r14
+
+function %insertlane_i32x4_mem_3(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 v1
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vlef %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_f32x4_mem_little_0(f32x4, i64) -> f32x4 {
+block0(v0: f32x4, v1: i64):
+ v2 = load.f32 little v1
+ v3 = insertlane.f32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vlebrf %v24, 0(%r2), 3
+; br %r14
+
+function %insertlane_i32x4_mem_little_3(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 little v1
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vlebrf %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i64x2_mem_0(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ v2 = extractlane.i64x2 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteg %v24, 0(%r2), 1
+; br %r14
+
+function %extractlane_i64x2_mem_1(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ v2 = extractlane.i64x2 v0, 1
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteg %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i64x2_mem_little_0(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ v2 = extractlane.i64x2 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrg %v24, 0(%r2), 1
+; br %r14
+
+function %extractlane_i64x2_mem_little_1(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ v2 = extractlane.i64x2 v0, 1
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrg %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i32x4_mem_0(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ v2 = extractlane.i32x4 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vstef %v24, 0(%r2), 3
+; br %r14
+
+function %extractlane_i32x4_mem_3(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ v2 = extractlane.i32x4 v0, 3
+ store v2, v1
+ return
+}
+
+; block0:
+; vstef %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i32x4_mem_little_0(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ v2 = extractlane.i32x4 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrf %v24, 0(%r2), 3
+; br %r14
+
+function %extractlane_i32x4_mem_little_3(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ v2 = extractlane.i32x4 v0, 3
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrf %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i16x8_mem_0(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ v2 = extractlane.i16x8 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteh %v24, 0(%r2), 7
+; br %r14
+
+function %extractlane_i16x8_mem_7(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ v2 = extractlane.i16x8 v0, 7
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteh %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i16x8_mem_little_0(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ v2 = extractlane.i16x8 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrh %v24, 0(%r2), 7
+; br %r14
+
+function %extractlane_i16x8_mem_little_7(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ v2 = extractlane.i16x8 v0, 7
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrh %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i8x16_mem_0(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ v2 = extractlane.i8x16 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteb %v24, 0(%r2), 15
+; br %r14
+
+function %extractlane_i8x16_mem_15(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ v2 = extractlane.i8x16 v0, 15
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteb %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i8x16_mem_little_0(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ v2 = extractlane.i8x16 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vsteb %v24, 0(%r2), 15
+; br %r14
+
+function %extractlane_i8x16_mem_little_15(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ v2 = extractlane.i8x16 v0, 15
+ store little v2, v1
+ return
+}
+
+; block0:
+; vsteb %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_f64x2_mem_0(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ v2 = extractlane.f64x2 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteg %v24, 0(%r2), 1
+; br %r14
+
+function %extractlane_f64x2_mem_1(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ v2 = extractlane.f64x2 v0, 1
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteg %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_f64x2_mem_little_0(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ v2 = extractlane.f64x2 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrg %v24, 0(%r2), 1
+; br %r14
+
+function %extractlane_f64x2_mem_little_1(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ v2 = extractlane.f64x2 v0, 1
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrg %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_f32x4_mem_0(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ v2 = extractlane.f32x4 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vstef %v24, 0(%r2), 3
+; br %r14
+
+function %extractlane_f32x4_mem_3(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ v2 = extractlane.f32x4 v0, 3
+ store v2, v1
+ return
+}
+
+; block0:
+; vstef %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_f32x4_mem_little_0(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ v2 = extractlane.f32x4 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrf %v24, 0(%r2), 3
+; br %r14
+
+function %extractlane_f32x4_mem_little_3(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ v2 = extractlane.f32x4 v0, 3
+ store little v2, v1
+ return
+}
+
+; block0:
+; vstebrf %v24, 0(%r2), 0
+; br %r14
+
+function %splat_i64x2_mem(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64 v0
+ v2 = splat.i64x2 v1
+ return v2
+}
+
+; block0:
+; vlrepg %v24, 0(%r2)
+; br %r14
+
+function %splat_i64x2_mem_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64 little v0
+ v2 = splat.i64x2 v1
+ return v2
+}
+
+; block0:
+; vlbrrepg %v24, 0(%r2)
+; br %r14
+
+function %splat_i32x4_mem(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32 v0
+ v2 = splat.i32x4 v1
+ return v2
+}
+
+; block0:
+; vlrepf %v24, 0(%r2)
+; br %r14
+
+function %splat_i32x4_mem_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32 little v0
+ v2 = splat.i32x4 v1
+ return v2
+}
+
+; block0:
+; vlbrrepf %v24, 0(%r2)
+; br %r14
+
+function %splat_i16x8_mem(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16 v0
+ v2 = splat.i16x8 v1
+ return v2
+}
+
+; block0:
+; vlreph %v24, 0(%r2)
+; br %r14
+
+function %splat_i16x8_mem_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16 little v0
+ v2 = splat.i16x8 v1
+ return v2
+}
+
+; block0:
+; vlbrreph %v24, 0(%r2)
+; br %r14
+
+function %splat_i8x16_mem(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8 v0
+ v2 = splat.i8x16 v1
+ return v2
+}
+
+; block0:
+; vlrepb %v24, 0(%r2)
+; br %r14
+
+function %splat_i8x16_mem_little(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8 little v0
+ v2 = splat.i8x16 v1
+ return v2
+}
+
+; block0:
+; vlrepb %v24, 0(%r2)
+; br %r14
+
+function %splat_f64x2_mem(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64 v0
+ v2 = splat.f64x2 v1
+ return v2
+}
+
+; block0:
+; vlrepg %v24, 0(%r2)
+; br %r14
+
+function %splat_f64x2_mem_little(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64 little v0
+ v2 = splat.f64x2 v1
+ return v2
+}
+
+; block0:
+; vlbrrepg %v24, 0(%r2)
+; br %r14
+
+function %splat_f32x4_mem(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32 v0
+ v2 = splat.f32x4 v1
+ return v2
+}
+
+; block0:
+; vlrepf %v24, 0(%r2)
+; br %r14
+
+function %splat_f32x4_mem_little(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32 little v0
+ v2 = splat.f32x4 v1
+ return v2
+}
+
+; block0:
+; vlbrrepf %v24, 0(%r2)
+; br %r14
+
+function %scalar_to_vector_i64x2_mem(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64 v0
+ v2 = scalar_to_vector.i64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleg %v24, 0(%r2), 1
+; br %r14
+
+function %scalar_to_vector_i64x2_mem_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64 little v0
+ v2 = scalar_to_vector.i64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vlebrg %v24, 0(%r2), 1
+; br %r14
+
+function %scalar_to_vector_i32x4_mem(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32 v0
+ v2 = scalar_to_vector.i32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vlef %v24, 0(%r2), 3
+; br %r14
+
+function %scalar_to_vector_i32x4_mem_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32 little v0
+ v2 = scalar_to_vector.i32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vlebrf %v24, 0(%r2), 3
+; br %r14
+
+function %scalar_to_vector_i16x8_mem(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16 v0
+ v2 = scalar_to_vector.i16x8 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleh %v24, 0(%r2), 7
+; br %r14
+
+function %scalar_to_vector_i16x8_mem_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16 little v0
+ v2 = scalar_to_vector.i16x8 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vlebrh %v24, 0(%r2), 7
+; br %r14
+
+function %scalar_to_vector_i8x16_mem(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8 v0
+ v2 = scalar_to_vector.i8x16 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleb %v24, 0(%r2), 15
+; br %r14
+
+function %scalar_to_vector_i8x16_mem_little(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8 little v0
+ v2 = scalar_to_vector.i8x16 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleb %v24, 0(%r2), 15
+; br %r14
+
+function %scalar_to_vector_f64x2_mem(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64 v0
+ v2 = scalar_to_vector.f64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleg %v24, 0(%r2), 1
+; br %r14
+
+function %scalar_to_vector_f64x2_mem_little(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64 little v0
+ v2 = scalar_to_vector.f64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vlebrg %v24, 0(%r2), 1
+; br %r14
+
+function %scalar_to_vector_f32x4_mem(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32 v0
+ v2 = scalar_to_vector.f32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vlef %v24, 0(%r2), 3
+; br %r14
+
+function %scalar_to_vector_f32x4_mem_little(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32 little v0
+ v2 = scalar_to_vector.f32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vlebrf %v24, 0(%r2), 3
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-lane.clif b/cranelift/filetests/filetests/isa/s390x/vec-lane.clif
new file mode 100644
index 000000000000..7efa4e3b719a
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-lane.clif
@@ -0,0 +1,1964 @@
+test compile precise-output
+target s390x
+
+function %insertlane_i64x2_0(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = insertlane.i64x2 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vlvgg %v24, %r2, 1
+; br %r14
+
+function %insertlane_i64x2_1(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = insertlane.i64x2 v0, v1, 1
+ return v2
+}
+
+; block0:
+; vlvgg %v24, %r2, 0
+; br %r14
+
+function %insertlane_i64x2_imm_0(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = iconst.i64 123
+ v2 = insertlane.i64x2 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vleig %v24, 123, 1
+; br %r14
+
+function %insertlane_i64x2_imm_1(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = iconst.i64 123
+ v2 = insertlane.i64x2 v0, v1, 1
+ return v2
+}
+
+; block0:
+; vleig %v24, 123, 0
+; br %r14
+
+function %insertlane_i64x2_lane_0_0(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = extractlane.i64x2 v1, 0
+ v3 = insertlane.i64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vpdi %v24, %v24, %v25, 1
+; br %r14
+
+function %insertlane_i64x2_lane_0_1(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = extractlane.i64x2 v1, 0
+ v3 = insertlane.i64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vpdi %v24, %v25, %v24, 5
+; br %r14
+
+function %insertlane_i64x2_lane_1_0(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = extractlane.i64x2 v1, 1
+ v3 = insertlane.i64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vpdi %v24, %v24, %v25, 0
+; br %r14
+
+function %insertlane_i64x2_lane_1_1(i64x2, i64x2) -> i64x2 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = extractlane.i64x2 v1, 1
+ v3 = insertlane.i64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vpdi %v24, %v25, %v24, 1
+; br %r14
+
+function %insertlane_i64x2_mem_0(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = load.i64 v1
+ v3 = insertlane.i64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleg %v24, 0(%r2), 1
+; br %r14
+
+function %insertlane_i64x2_mem_1(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = load.i64 v1
+ v3 = insertlane.i64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vleg %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i64x2_mem_little_0(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = load.i64 little v1
+ v3 = insertlane.i64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; lrvg %r3, 0(%r2)
+; vlvgg %v24, %r3, 1
+; br %r14
+
+function %insertlane_i64x2_mem_little_1(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = load.i64 little v1
+ v3 = insertlane.i64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; lrvg %r3, 0(%r2)
+; vlvgg %v24, %r3, 0
+; br %r14
+
+function %insertlane_i32x4_0(i32x4, i32) -> i32x4 {
+block0(v0: i32x4, v1: i32):
+ v2 = insertlane.i32x4 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vlvgf %v24, %r2, 3
+; br %r14
+
+function %insertlane_i32x4_3(i32x4, i32) -> i32x4 {
+block0(v0: i32x4, v1: i32):
+ v2 = insertlane.i32x4 v0, v1, 3
+ return v2
+}
+
+; block0:
+; vlvgf %v24, %r2, 0
+; br %r14
+
+function %insertlane_i32x4_imm_0(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = iconst.i32 123
+ v2 = insertlane.i32x4 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vleif %v24, 123, 3
+; br %r14
+
+function %insertlane_i32x4_imm_3(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = iconst.i32 123
+ v2 = insertlane.i32x4 v0, v1, 3
+ return v2
+}
+
+; block0:
+; vleif %v24, 123, 0
+; br %r14
+
+function %insertlane_i32x4_lane_0_0(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = extractlane.i32x4 v1, 0
+ v3 = insertlane.i32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vgbm %v5, 15
+; vsel %v24, %v25, %v24, %v5
+; br %r14
+
+function %insertlane_i32x4_lane_0_3(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = extractlane.i32x4 v1, 0
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vrepf %v5, %v25, 3
+; vgbm %v7, 61440
+; vsel %v24, %v5, %v24, %v7
+; br %r14
+
+function %insertlane_i32x4_lane_3_0(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = extractlane.i32x4 v1, 3
+ v3 = insertlane.i32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vrepf %v5, %v25, 0
+; vgbm %v7, 15
+; vsel %v24, %v5, %v24, %v7
+; br %r14
+
+function %insertlane_i32x4_lane_3_3(i32x4, i32x4) -> i32x4 {
+block0(v0: i32x4, v1: i32x4):
+ v2 = extractlane.i32x4 v1, 3
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vgbm %v5, 61440
+; vsel %v24, %v25, %v24, %v5
+; br %r14
+
+function %insertlane_i32x4_mem_0(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 v1
+ v3 = insertlane.i32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vlef %v24, 0(%r2), 3
+; br %r14
+
+function %insertlane_i32x4_mem_3(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 v1
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vlef %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i32x4_mem_little_0(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 little v1
+ v3 = insertlane.i32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; lrv %r3, 0(%r2)
+; vlvgf %v24, %r3, 3
+; br %r14
+
+function %insertlane_i32x4_mem_little_3(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 little v1
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; lrv %r3, 0(%r2)
+; vlvgf %v24, %r3, 0
+; br %r14
+
+function %insertlane_i16x8_0(i16x8, i16) -> i16x8 {
+block0(v0: i16x8, v1: i16):
+ v2 = insertlane.i16x8 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vlvgh %v24, %r2, 7
+; br %r14
+
+function %insertlane_i16x8_7(i16x8, i16) -> i16x8 {
+block0(v0: i16x8, v1: i16):
+ v2 = insertlane.i16x8 v0, v1, 7
+ return v2
+}
+
+; block0:
+; vlvgh %v24, %r2, 0
+; br %r14
+
+function %insertlane_i16x8_imm_0(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = iconst.i16 123
+ v2 = insertlane.i16x8 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vleih %v24, 123, 7
+; br %r14
+
+function %insertlane_i16x8_imm_7(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = iconst.i16 123
+ v2 = insertlane.i16x8 v0, v1, 7
+ return v2
+}
+
+; block0:
+; vleih %v24, 123, 0
+; br %r14
+
+function %insertlane_i16x8_lane_0_0(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = extractlane.i16x8 v1, 0
+ v3 = insertlane.i16x8 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vgbm %v5, 3
+; vsel %v24, %v25, %v24, %v5
+; br %r14
+
+function %insertlane_i16x8_lane_0_7(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = extractlane.i16x8 v1, 0
+ v3 = insertlane.i16x8 v0, v2, 7
+ return v3
+}
+
+; block0:
+; vreph %v5, %v25, 7
+; vgbm %v7, 49152
+; vsel %v24, %v5, %v24, %v7
+; br %r14
+
+function %insertlane_i16x8_lane_7_0(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = extractlane.i16x8 v1, 7
+ v3 = insertlane.i16x8 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vreph %v5, %v25, 0
+; vgbm %v7, 3
+; vsel %v24, %v5, %v24, %v7
+; br %r14
+
+function %insertlane_i16x8_lane_7_7(i16x8, i16x8) -> i16x8 {
+block0(v0: i16x8, v1: i16x8):
+ v2 = extractlane.i16x8 v1, 7
+ v3 = insertlane.i16x8 v0, v2, 7
+ return v3
+}
+
+; block0:
+; vgbm %v5, 49152
+; vsel %v24, %v25, %v24, %v5
+; br %r14
+
+function %insertlane_i16x8_mem_0(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+ v2 = load.i16 v1
+ v3 = insertlane.i16x8 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleh %v24, 0(%r2), 7
+; br %r14
+
+function %insertlane_i16x8_mem_7(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+ v2 = load.i16 v1
+ v3 = insertlane.i16x8 v0, v2, 7
+ return v3
+}
+
+; block0:
+; vleh %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i16x8_mem_little_0(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+ v2 = load.i16 little v1
+ v3 = insertlane.i16x8 v0, v2, 0
+ return v3
+}
+
+; block0:
+; lrvh %r3, 0(%r2)
+; vlvgh %v24, %r3, 7
+; br %r14
+
+function %insertlane_i16x8_mem_little_7(i16x8, i64) -> i16x8 {
+block0(v0: i16x8, v1: i64):
+ v2 = load.i16 little v1
+ v3 = insertlane.i16x8 v0, v2, 7
+ return v3
+}
+
+; block0:
+; lrvh %r3, 0(%r2)
+; vlvgh %v24, %r3, 0
+; br %r14
+
+function %insertlane_i8x16_0(i8x16, i8) -> i8x16 {
+block0(v0: i8x16, v1: i8):
+ v2 = insertlane.i8x16 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vlvgb %v24, %r2, 15
+; br %r14
+
+function %insertlane_i8x16_15(i8x16, i8) -> i8x16 {
+block0(v0: i8x16, v1: i8):
+ v2 = insertlane.i8x16 v0, v1, 15
+ return v2
+}
+
+; block0:
+; vlvgb %v24, %r2, 0
+; br %r14
+
+function %insertlane_i8x16_imm_0(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = iconst.i8 123
+ v2 = insertlane.i8x16 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vleib %v24, 123, 15
+; br %r14
+
+function %insertlane_i8x16_imm_15(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = iconst.i8 123
+ v2 = insertlane.i8x16 v0, v1, 15
+ return v2
+}
+
+; block0:
+; vleib %v24, 123, 0
+; br %r14
+
+function %insertlane_i8x16_lane_0_0(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = extractlane.i8x16 v1, 0
+ v3 = insertlane.i8x16 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vgbm %v5, 1
+; vsel %v24, %v25, %v24, %v5
+; br %r14
+
+function %insertlane_i8x16_lane_0_15(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = extractlane.i8x16 v1, 0
+ v3 = insertlane.i8x16 v0, v2, 15
+ return v3
+}
+
+; block0:
+; vrepb %v5, %v25, 15
+; vgbm %v7, 32768
+; vsel %v24, %v5, %v24, %v7
+; br %r14
+
+function %insertlane_i8x16_lane_15_0(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = extractlane.i8x16 v1, 15
+ v3 = insertlane.i8x16 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vrepb %v5, %v25, 0
+; vgbm %v7, 1
+; vsel %v24, %v5, %v24, %v7
+; br %r14
+
+function %insertlane_i8x16_lane_15_15(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = extractlane.i8x16 v1, 15
+ v3 = insertlane.i8x16 v0, v2, 15
+ return v3
+}
+
+; block0:
+; vgbm %v5, 32768
+; vsel %v24, %v25, %v24, %v5
+; br %r14
+
+function %insertlane_i8x16_mem_0(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+ v2 = load.i8 v1
+ v3 = insertlane.i8x16 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleb %v24, 0(%r2), 15
+; br %r14
+
+function %insertlane_i8x16_mem_15(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+ v2 = load.i8 v1
+ v3 = insertlane.i8x16 v0, v2, 15
+ return v3
+}
+
+; block0:
+; vleb %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_i8x16_mem_little_0(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+ v2 = load.i8 little v1
+ v3 = insertlane.i8x16 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleb %v24, 0(%r2), 15
+; br %r14
+
+function %insertlane_i8x16_mem_little_15(i8x16, i64) -> i8x16 {
+block0(v0: i8x16, v1: i64):
+ v2 = load.i8 little v1
+ v3 = insertlane.i8x16 v0, v2, 15
+ return v3
+}
+
+; block0:
+; vleb %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_f64x2_0(f64x2, f64) -> f64x2 {
+block0(v0: f64x2, v1: f64):
+ v2 = insertlane.f64x2 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vpdi %v24, %v24, %v0, 0
+; br %r14
+
+function %insertlane_f64x2_1(f64x2, f64) -> f64x2 {
+block0(v0: f64x2, v1: f64):
+ v2 = insertlane.f64x2 v0, v1, 1
+ return v2
+}
+
+; block0:
+; vpdi %v24, %v0, %v24, 1
+; br %r14
+
+function %insertlane_f64x2_lane_0_0(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = extractlane.f64x2 v1, 0
+ v3 = insertlane.f64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vpdi %v24, %v24, %v25, 1
+; br %r14
+
+function %insertlane_f64x2_lane_0_1(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = extractlane.f64x2 v1, 0
+ v3 = insertlane.f64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vpdi %v24, %v25, %v24, 5
+; br %r14
+
+function %insertlane_f64x2_lane_1_0(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = extractlane.f64x2 v1, 1
+ v3 = insertlane.f64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vpdi %v24, %v24, %v25, 0
+; br %r14
+
+function %insertlane_f64x2_lane_1_1(f64x2, f64x2) -> f64x2 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = extractlane.f64x2 v1, 1
+ v3 = insertlane.f64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vpdi %v24, %v25, %v24, 1
+; br %r14
+
+function %insertlane_f64x2_mem_0(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+ v2 = load.f64 v1
+ v3 = insertlane.f64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vleg %v24, 0(%r2), 1
+; br %r14
+
+function %insertlane_f64x2_mem_1(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+ v2 = load.f64 v1
+ v3 = insertlane.f64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; vleg %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_f64x2_mem_little_0(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+ v2 = load.f64 little v1
+ v3 = insertlane.f64x2 v0, v2, 0
+ return v3
+}
+
+; block0:
+; lrvg %r3, 0(%r2)
+; vlvgg %v24, %r3, 1
+; br %r14
+
+function %insertlane_f64x2_mem_little_1(f64x2, i64) -> f64x2 {
+block0(v0: f64x2, v1: i64):
+ v2 = load.f64 little v1
+ v3 = insertlane.f64x2 v0, v2, 1
+ return v3
+}
+
+; block0:
+; lrvg %r3, 0(%r2)
+; vlvgg %v24, %r3, 0
+; br %r14
+
+function %insertlane_f32x4_0(f32x4, f32) -> f32x4 {
+block0(v0: f32x4, v1: f32):
+ v2 = insertlane.f32x4 v0, v1, 0
+ return v2
+}
+
+; block0:
+; vrepf %v5, %v0, 0
+; vgbm %v7, 15
+; vsel %v24, %v5, %v24, %v7
+; br %r14
+
+function %insertlane_f32x4_3(f32x4, f32) -> f32x4 {
+block0(v0: f32x4, v1: f32):
+ v2 = insertlane.f32x4 v0, v1, 3
+ return v2
+}
+
+; block0:
+; vgbm %v5, 61440
+; vsel %v24, %v0, %v24, %v5
+; br %r14
+
+function %insertlane_f32x4_lane_0_0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = extractlane.f32x4 v1, 0
+ v3 = insertlane.f32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vgbm %v5, 15
+; vsel %v24, %v25, %v24, %v5
+; br %r14
+
+function %insertlane_f32x4_lane_0_3(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = extractlane.f32x4 v1, 0
+ v3 = insertlane.f32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vrepf %v5, %v25, 3
+; vgbm %v7, 61440
+; vsel %v24, %v5, %v24, %v7
+; br %r14
+
+function %insertlane_f32x4_lane_3_0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = extractlane.f32x4 v1, 3
+ v3 = insertlane.f32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vrepf %v5, %v25, 0
+; vgbm %v7, 15
+; vsel %v24, %v5, %v24, %v7
+; br %r14
+
+function %insertlane_f32x4_lane_3_3(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+ v2 = extractlane.f32x4 v1, 3
+ v3 = insertlane.f32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vgbm %v5, 61440
+; vsel %v24, %v25, %v24, %v5
+; br %r14
+
+function %insertlane_f32x4_mem_0(f32x4, i64) -> f32x4 {
+block0(v0: f32x4, v1: i64):
+ v2 = load.f32 v1
+ v3 = insertlane.f32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; vlef %v24, 0(%r2), 3
+; br %r14
+
+function %insertlane_i32x4_mem_3(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 v1
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; vlef %v24, 0(%r2), 0
+; br %r14
+
+function %insertlane_f32x4_mem_little_0(f32x4, i64) -> f32x4 {
+block0(v0: f32x4, v1: i64):
+ v2 = load.f32 little v1
+ v3 = insertlane.f32x4 v0, v2, 0
+ return v3
+}
+
+; block0:
+; lrv %r3, 0(%r2)
+; vlvgf %v24, %r3, 3
+; br %r14
+
+function %insertlane_i32x4_mem_little_3(i32x4, i64) -> i32x4 {
+block0(v0: i32x4, v1: i64):
+ v2 = load.i32 little v1
+ v3 = insertlane.i32x4 v0, v2, 3
+ return v3
+}
+
+; block0:
+; lrv %r3, 0(%r2)
+; vlvgf %v24, %r3, 0
+; br %r14
+
+function %extractlane_i64x2_0(i64x2) -> i64 {
+block0(v0: i64x2):
+ v1 = extractlane.i64x2 v0, 0
+ return v1
+}
+
+; block0:
+; vlgvg %r2, %v24, 1
+; br %r14
+
+function %extractlane_i64x2_1(i64x2) -> i64 {
+block0(v0: i64x2):
+ v1 = extractlane.i64x2 v0, 1
+ return v1
+}
+
+; block0:
+; vlgvg %r2, %v24, 0
+; br %r14
+
+function %extractlane_i64x2_mem_0(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ v2 = extractlane.i64x2 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteg %v24, 0(%r2), 1
+; br %r14
+
+function %extractlane_i64x2_mem_1(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ v2 = extractlane.i64x2 v0, 1
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteg %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i64x2_mem_little_0(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ v2 = extractlane.i64x2 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 1
+; strvg %r3, 0(%r2)
+; br %r14
+
+function %extractlane_i64x2_mem_little_1(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ v2 = extractlane.i64x2 v0, 1
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 0
+; strvg %r3, 0(%r2)
+; br %r14
+
+function %extractlane_i32x4_0(i32x4) -> i32 {
+block0(v0: i32x4):
+ v1 = extractlane.i32x4 v0, 0
+ return v1
+}
+
+; block0:
+; vlgvf %r2, %v24, 3
+; br %r14
+
+function %extractlane_i32x4_3(i32x4) -> i32 {
+block0(v0: i32x4):
+ v1 = extractlane.i32x4 v0, 3
+ return v1
+}
+
+; block0:
+; vlgvf %r2, %v24, 0
+; br %r14
+
+function %extractlane_i32x4_mem_0(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ v2 = extractlane.i32x4 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vstef %v24, 0(%r2), 3
+; br %r14
+
+function %extractlane_i32x4_mem_3(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ v2 = extractlane.i32x4 v0, 3
+ store v2, v1
+ return
+}
+
+; block0:
+; vstef %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i32x4_mem_little_0(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ v2 = extractlane.i32x4 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvf %r3, %v24, 3
+; strv %r3, 0(%r2)
+; br %r14
+
+function %extractlane_i32x4_mem_little_3(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ v2 = extractlane.i32x4 v0, 3
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvf %r3, %v24, 0
+; strv %r3, 0(%r2)
+; br %r14
+
+function %extractlane_i16x8_0(i16x8) -> i16 {
+block0(v0: i16x8):
+ v1 = extractlane.i16x8 v0, 0
+ return v1
+}
+
+; block0:
+; vlgvh %r2, %v24, 7
+; br %r14
+
+function %extractlane_i16x8_7(i16x8) -> i16 {
+block0(v0: i16x8):
+ v1 = extractlane.i16x8 v0, 7
+ return v1
+}
+
+; block0:
+; vlgvh %r2, %v24, 0
+; br %r14
+
+function %extractlane_i16x8_mem_0(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ v2 = extractlane.i16x8 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteh %v24, 0(%r2), 7
+; br %r14
+
+function %extractlane_i16x8_mem_7(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ v2 = extractlane.i16x8 v0, 7
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteh %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i16x8_mem_little_0(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ v2 = extractlane.i16x8 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvh %r3, %v24, 7
+; strvh %r3, 0(%r2)
+; br %r14
+
+function %extractlane_i16x8_mem_little_7(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ v2 = extractlane.i16x8 v0, 7
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvh %r3, %v24, 0
+; strvh %r3, 0(%r2)
+; br %r14
+
+function %extractlane_i8x16_0(i8x16) -> i8 {
+block0(v0: i8x16):
+ v1 = extractlane.i8x16 v0, 0
+ return v1
+}
+
+; block0:
+; vlgvb %r2, %v24, 15
+; br %r14
+
+function %extractlane_i8x16_15(i8x16) -> i8 {
+block0(v0: i8x16):
+ v1 = extractlane.i8x16 v0, 15
+ return v1
+}
+
+; block0:
+; vlgvb %r2, %v24, 0
+; br %r14
+
+function %extractlane_i8x16_mem_0(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ v2 = extractlane.i8x16 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteb %v24, 0(%r2), 15
+; br %r14
+
+function %extractlane_i8x16_mem_15(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ v2 = extractlane.i8x16 v0, 15
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteb %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_i8x16_mem_little_0(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ v2 = extractlane.i8x16 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vsteb %v24, 0(%r2), 15
+; br %r14
+
+function %extractlane_i8x16_mem_little_15(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ v2 = extractlane.i8x16 v0, 15
+ store little v2, v1
+ return
+}
+
+; block0:
+; vsteb %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_f64x2_0(f64x2) -> f64 {
+block0(v0: f64x2):
+ v1 = extractlane.f64x2 v0, 0
+ return v1
+}
+
+; block0:
+; vrepg %v0, %v24, 1
+; br %r14
+
+function %extractlane_f64x2_1(f64x2) -> f64 {
+block0(v0: f64x2):
+ v1 = extractlane.f64x2 v0, 1
+ return v1
+}
+
+; block0:
+; vrepg %v0, %v24, 0
+; br %r14
+
+function %extractlane_f64x2_mem_0(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ v2 = extractlane.f64x2 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteg %v24, 0(%r2), 1
+; br %r14
+
+function %extractlane_f64x2_mem_1(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ v2 = extractlane.f64x2 v0, 1
+ store v2, v1
+ return
+}
+
+; block0:
+; vsteg %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_f64x2_mem_little_0(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ v2 = extractlane.f64x2 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 1
+; strvg %r3, 0(%r2)
+; br %r14
+
+function %extractlane_f64x2_mem_little_1(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ v2 = extractlane.f64x2 v0, 1
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 0
+; strvg %r3, 0(%r2)
+; br %r14
+
+function %extractlane_f32x4_0(f32x4) -> f32 {
+block0(v0: f32x4):
+ v1 = extractlane.f32x4 v0, 0
+ return v1
+}
+
+; block0:
+; vrepf %v0, %v24, 3
+; br %r14
+
+function %extractlane_f32x4_3(f32x4) -> f32 {
+block0(v0: f32x4):
+ v1 = extractlane.f32x4 v0, 3
+ return v1
+}
+
+; block0:
+; vrepf %v0, %v24, 0
+; br %r14
+
+function %extractlane_f32x4_mem_0(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ v2 = extractlane.f32x4 v0, 0
+ store v2, v1
+ return
+}
+
+; block0:
+; vstef %v24, 0(%r2), 3
+; br %r14
+
+function %extractlane_f32x4_mem_3(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ v2 = extractlane.f32x4 v0, 3
+ store v2, v1
+ return
+}
+
+; block0:
+; vstef %v24, 0(%r2), 0
+; br %r14
+
+function %extractlane_f32x4_mem_little_0(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ v2 = extractlane.f32x4 v0, 0
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvf %r3, %v24, 3
+; strv %r3, 0(%r2)
+; br %r14
+
+function %extractlane_f32x4_mem_little_3(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ v2 = extractlane.f32x4 v0, 3
+ store little v2, v1
+ return
+}
+
+; block0:
+; vlgvf %r3, %v24, 0
+; strv %r3, 0(%r2)
+; br %r14
+
+function %splat_i64x2(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = splat.i64x2 v0
+ return v1
+}
+
+; block0:
+; ldgr %f3, %r2
+; vrepg %v24, %v3, 0
+; br %r14
+
+function %splat_i64x2_imm() -> i64x2 {
+block0:
+ v0 = iconst.i64 123
+ v1 = splat.i64x2 v0
+ return v1
+}
+
+; block0:
+; vrepig %v24, 123
+; br %r14
+
+function %splat_i64x2_lane_0(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = extractlane.i64x2 v0, 0
+ v2 = splat.i64x2 v1
+ return v2
+}
+
+; block0:
+; vrepg %v24, %v24, 1
+; br %r14
+
+function %splat_i64x2_lane_1(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = extractlane.i64x2 v0, 1
+ v2 = splat.i64x2 v1
+ return v2
+}
+
+; block0:
+; vrepg %v24, %v24, 0
+; br %r14
+
+function %splat_i64x2_mem(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64 v0
+ v2 = splat.i64x2 v1
+ return v2
+}
+
+; block0:
+; vlrepg %v24, 0(%r2)
+; br %r14
+
+function %splat_i64x2_mem_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64 little v0
+ v2 = splat.i64x2 v1
+ return v2
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; ldgr %f5, %r5
+; vrepg %v24, %v5, 0
+; br %r14
+
+function %splat_i32x4(i32) -> i32x4 {
+block0(v0: i32):
+ v1 = splat.i32x4 v0
+ return v1
+}
+
+; block0:
+; vlvgf %v3, %r2, 0
+; vrepf %v24, %v3, 0
+; br %r14
+
+function %splat_i32x4_imm() -> i32x4 {
+block0:
+ v0 = iconst.i32 123
+ v1 = splat.i32x4 v0
+ return v1
+}
+
+; block0:
+; vrepif %v24, 123
+; br %r14
+
+function %splat_i32x4_lane_0(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = extractlane.i32x4 v0, 0
+ v2 = splat.i32x4 v1
+ return v2
+}
+
+; block0:
+; vrepf %v24, %v24, 3
+; br %r14
+
+function %splat_i32x4_lane_3(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = extractlane.i32x4 v0, 3
+ v2 = splat.i32x4 v1
+ return v2
+}
+
+; block0:
+; vrepf %v24, %v24, 0
+; br %r14
+
+function %splat_i32x4_mem(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32 v0
+ v2 = splat.i32x4 v1
+ return v2
+}
+
+; block0:
+; vlrepf %v24, 0(%r2)
+; br %r14
+
+function %splat_i32x4_mem_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32 little v0
+ v2 = splat.i32x4 v1
+ return v2
+}
+
+; block0:
+; lrv %r5, 0(%r2)
+; vlvgf %v5, %r5, 0
+; vrepf %v24, %v5, 0
+; br %r14
+
+function %splat_i16x8(i16) -> i16x8 {
+block0(v0: i16):
+ v1 = splat.i16x8 v0
+ return v1
+}
+
+; block0:
+; vlvgh %v3, %r2, 0
+; vreph %v24, %v3, 0
+; br %r14
+
+function %splat_i16x8_imm() -> i16x8 {
+block0:
+ v0 = iconst.i16 123
+ v1 = splat.i16x8 v0
+ return v1
+}
+
+; block0:
+; vrepih %v24, 123
+; br %r14
+
+function %splat_i16x8_lane_0(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = extractlane.i16x8 v0, 0
+ v2 = splat.i16x8 v1
+ return v2
+}
+
+; block0:
+; vreph %v24, %v24, 7
+; br %r14
+
+function %splat_i16x8_lane_7(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = extractlane.i16x8 v0, 7
+ v2 = splat.i16x8 v1
+ return v2
+}
+
+; block0:
+; vreph %v24, %v24, 0
+; br %r14
+
+function %splat_i16x8_mem(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16 v0
+ v2 = splat.i16x8 v1
+ return v2
+}
+
+; block0:
+; vlreph %v24, 0(%r2)
+; br %r14
+
+function %splat_i16x8_mem_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16 little v0
+ v2 = splat.i16x8 v1
+ return v2
+}
+
+; block0:
+; lrvh %r5, 0(%r2)
+; vlvgh %v5, %r5, 0
+; vreph %v24, %v5, 0
+; br %r14
+
+function %splat_i8x16(i8) -> i8x16 {
+block0(v0: i8):
+ v1 = splat.i8x16 v0
+ return v1
+}
+
+; block0:
+; vlvgb %v3, %r2, 0
+; vrepb %v24, %v3, 0
+; br %r14
+
+function %splat_i8x16_imm() -> i8x16 {
+block0:
+ v0 = iconst.i8 123
+ v1 = splat.i8x16 v0
+ return v1
+}
+
+; block0:
+; vrepib %v24, 123
+; br %r14
+
+function %splat_i8x16_lane_0(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = extractlane.i8x16 v0, 0
+ v2 = splat.i8x16 v1
+ return v2
+}
+
+; block0:
+; vrepb %v24, %v24, 15
+; br %r14
+
+function %splat_i8x16_lane_15(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = extractlane.i8x16 v0, 15
+ v2 = splat.i8x16 v1
+ return v2
+}
+
+; block0:
+; vrepb %v24, %v24, 0
+; br %r14
+
+function %splat_i8x16_mem(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8 v0
+ v2 = splat.i8x16 v1
+ return v2
+}
+
+; block0:
+; vlrepb %v24, 0(%r2)
+; br %r14
+
+function %splat_i8x16_mem_little(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8 little v0
+ v2 = splat.i8x16 v1
+ return v2
+}
+
+; block0:
+; vlrepb %v24, 0(%r2)
+; br %r14
+
+function %splat_f64x2(f64) -> f64x2 {
+block0(v0: f64):
+ v1 = splat.f64x2 v0
+ return v1
+}
+
+; block0:
+; vrepg %v24, %v0, 0
+; br %r14
+
+function %splat_f64x2_lane_0(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = extractlane.f64x2 v0, 0
+ v2 = splat.f64x2 v1
+ return v2
+}
+
+; block0:
+; vrepg %v24, %v24, 1
+; br %r14
+
+function %splat_f64x2_lane_1(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = extractlane.f64x2 v0, 1
+ v2 = splat.f64x2 v1
+ return v2
+}
+
+; block0:
+; vrepg %v24, %v24, 0
+; br %r14
+
+function %splat_f64x2_mem(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64 v0
+ v2 = splat.f64x2 v1
+ return v2
+}
+
+; block0:
+; vlrepg %v24, 0(%r2)
+; br %r14
+
+function %splat_f64x2_mem_little(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64 little v0
+ v2 = splat.f64x2 v1
+ return v2
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; ldgr %f5, %r5
+; vrepg %v24, %v5, 0
+; br %r14
+
+function %splat_f32x4(f32) -> f32x4 {
+block0(v0: f32):
+ v1 = splat.f32x4 v0
+ return v1
+}
+
+; block0:
+; vrepf %v24, %v0, 0
+; br %r14
+
+function %splat_f32x4_lane_0(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = extractlane.f32x4 v0, 0
+ v2 = splat.f32x4 v1
+ return v2
+}
+
+; block0:
+; vrepf %v24, %v24, 3
+; br %r14
+
+function %splat_i32x4_lane_3(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = extractlane.i32x4 v0, 3
+ v2 = splat.i32x4 v1
+ return v2
+}
+
+; block0:
+; vrepf %v24, %v24, 0
+; br %r14
+
+function %splat_f32x4_mem(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32 v0
+ v2 = splat.f32x4 v1
+ return v2
+}
+
+; block0:
+; vlrepf %v24, 0(%r2)
+; br %r14
+
+function %splat_f32x4_mem_little(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32 little v0
+ v2 = splat.f32x4 v1
+ return v2
+}
+
+; block0:
+; lrv %r5, 0(%r2)
+; vlvgf %v5, %r5, 0
+; vrepf %v24, %v5, 0
+; br %r14
+
+function %scalar_to_vector_i64x2(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = scalar_to_vector.i64x2 v0
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; vlvgg %v24, %r2, 1
+; br %r14
+
+function %scalar_to_vector_i64x2_imm() -> i64x2 {
+block0:
+ v0 = iconst.i64 123
+ v1 = scalar_to_vector.i64x2 v0
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; vleig %v24, 123, 1
+; br %r14
+
+function %scalar_to_vector_i64x2_lane_0(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = extractlane.i64x2 v0, 0
+ v2 = scalar_to_vector.i64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v3, 0
+; vpdi %v24, %v3, %v24, 1
+; br %r14
+
+function %scalar_to_vector_i64x2_lane_1(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = extractlane.i64x2 v0, 1
+ v2 = scalar_to_vector.i64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v3, 0
+; vpdi %v24, %v3, %v24, 0
+; br %r14
+
+function %scalar_to_vector_i64x2_mem(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64 v0
+ v2 = scalar_to_vector.i64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleg %v24, 0(%r2), 1
+; br %r14
+
+function %scalar_to_vector_i64x2_mem_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64 little v0
+ v2 = scalar_to_vector.i64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; lrvg %r3, 0(%r2)
+; vlvgg %v24, %r3, 1
+; br %r14
+
+function %scalar_to_vector_i32x4(i32) -> i32x4 {
+block0(v0: i32):
+ v1 = scalar_to_vector.i32x4 v0
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; vlvgf %v24, %r2, 3
+; br %r14
+
+function %scalar_to_vector_i32x4_imm() -> i32x4 {
+block0:
+ v0 = iconst.i32 123
+ v1 = scalar_to_vector.i32x4 v0
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; vleif %v24, 123, 3
+; br %r14
+
+function %scalar_to_vector_i32x4_lane_0(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = extractlane.i32x4 v0, 0
+ v2 = scalar_to_vector.i32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v3, 15
+; vn %v24, %v24, %v3
+; br %r14
+
+function %scalar_to_vector_i32x4_lane_3(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = extractlane.i32x4 v0, 3
+ v2 = scalar_to_vector.i32x4 v1
+ return v2
+}
+
+; block0:
+; vrepf %v3, %v24, 0
+; vgbm %v5, 15
+; vn %v24, %v3, %v5
+; br %r14
+
+function %scalar_to_vector_i32x4_mem(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32 v0
+ v2 = scalar_to_vector.i32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vlef %v24, 0(%r2), 3
+; br %r14
+
+function %scalar_to_vector_i32x4_mem_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32 little v0
+ v2 = scalar_to_vector.i32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; lrv %r3, 0(%r2)
+; vlvgf %v24, %r3, 3
+; br %r14
+
+function %scalar_to_vector_i16x8(i16) -> i16x8 {
+block0(v0: i16):
+ v1 = scalar_to_vector.i16x8 v0
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; vlvgh %v24, %r2, 7
+; br %r14
+
+function %scalar_to_vector_i16x8_imm() -> i16x8 {
+block0:
+ v0 = iconst.i16 123
+ v1 = scalar_to_vector.i16x8 v0
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; vleih %v24, 123, 7
+; br %r14
+
+function %scalar_to_vector_i16x8_lane_0(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = extractlane.i16x8 v0, 0
+ v2 = scalar_to_vector.i16x8 v1
+ return v2
+}
+
+; block0:
+; vgbm %v3, 3
+; vn %v24, %v24, %v3
+; br %r14
+
+function %scalar_to_vector_i16x8_lane_7(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = extractlane.i16x8 v0, 7
+ v2 = scalar_to_vector.i16x8 v1
+ return v2
+}
+
+; block0:
+; vreph %v3, %v24, 0
+; vgbm %v5, 3
+; vn %v24, %v3, %v5
+; br %r14
+
+function %scalar_to_vector_i16x8_mem(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16 v0
+ v2 = scalar_to_vector.i16x8 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleh %v24, 0(%r2), 7
+; br %r14
+
+function %scalar_to_vector_i16x8_mem_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16 little v0
+ v2 = scalar_to_vector.i16x8 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; lrvh %r3, 0(%r2)
+; vlvgh %v24, %r3, 7
+; br %r14
+
+function %scalar_to_vector_i8x16(i8) -> i8x16 {
+block0(v0: i8):
+ v1 = scalar_to_vector.i8x16 v0
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; vlvgb %v24, %r2, 15
+; br %r14
+
+function %scalar_to_vector_i8x16_imm() -> i8x16 {
+block0:
+ v0 = iconst.i8 123
+ v1 = scalar_to_vector.i8x16 v0
+ return v1
+}
+
+; block0:
+; vgbm %v24, 0
+; vleib %v24, 123, 15
+; br %r14
+
+function %scalar_to_vector_i8x16_lane_0(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = extractlane.i8x16 v0, 0
+ v2 = scalar_to_vector.i8x16 v1
+ return v2
+}
+
+; block0:
+; vgbm %v3, 1
+; vn %v24, %v24, %v3
+; br %r14
+
+function %scalar_to_vector_i8x16_lane_15(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = extractlane.i8x16 v0, 15
+ v2 = scalar_to_vector.i8x16 v1
+ return v2
+}
+
+; block0:
+; vrepb %v3, %v24, 0
+; vgbm %v5, 1
+; vn %v24, %v3, %v5
+; br %r14
+
+function %scalar_to_vector_i8x16_mem(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8 v0
+ v2 = scalar_to_vector.i8x16 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleb %v24, 0(%r2), 15
+; br %r14
+
+function %scalar_to_vector_i8x16_mem_little(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8 little v0
+ v2 = scalar_to_vector.i8x16 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleb %v24, 0(%r2), 15
+; br %r14
+
+function %scalar_to_vector_f64x2(f64) -> f64x2 {
+block0(v0: f64):
+ v1 = scalar_to_vector.f64x2 v0
+ return v1
+}
+
+; block0:
+; vgbm %v3, 0
+; vpdi %v24, %v3, %v0, 0
+; br %r14
+
+function %scalar_to_vector_f64x2_lane_0(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = extractlane.f64x2 v0, 0
+ v2 = scalar_to_vector.f64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v3, 0
+; vpdi %v24, %v3, %v24, 1
+; br %r14
+
+function %scalar_to_vector_f64x2_lane_1(f64x2) -> f64x2 {
+block0(v0: f64x2):
+ v1 = extractlane.f64x2 v0, 1
+ v2 = scalar_to_vector.f64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v3, 0
+; vpdi %v24, %v3, %v24, 0
+; br %r14
+
+function %scalar_to_vector_f64x2_mem(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64 v0
+ v2 = scalar_to_vector.f64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vleg %v24, 0(%r2), 1
+; br %r14
+
+function %scalar_to_vector_f64x2_mem_little(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64 little v0
+ v2 = scalar_to_vector.f64x2 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; lrvg %r3, 0(%r2)
+; vlvgg %v24, %r3, 1
+; br %r14
+
+function %scalar_to_vector_f32x4(f32) -> f32x4 {
+block0(v0: f32):
+ v1 = scalar_to_vector.f32x4 v0
+ return v1
+}
+
+; block0:
+; vrepf %v3, %v0, 0
+; vgbm %v5, 15
+; vn %v24, %v3, %v5
+; br %r14
+
+function %scalar_to_vector_f32x4_lane_0(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = extractlane.f32x4 v0, 0
+ v2 = scalar_to_vector.f32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v3, 15
+; vn %v24, %v24, %v3
+; br %r14
+
+function %scalar_to_vector_f32x4_lane_3(f32x4) -> f32x4 {
+block0(v0: f32x4):
+ v1 = extractlane.f32x4 v0, 3
+ v2 = scalar_to_vector.f32x4 v1
+ return v2
+}
+
+; block0:
+; vrepf %v3, %v24, 0
+; vgbm %v5, 15
+; vn %v24, %v3, %v5
+; br %r14
+
+function %scalar_to_vector_f32x4_mem(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32 v0
+ v2 = scalar_to_vector.f32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; vlef %v24, 0(%r2), 3
+; br %r14
+
+function %scalar_to_vector_f32x4_mem_little(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32 little v0
+ v2 = scalar_to_vector.f32x4 v1
+ return v2
+}
+
+; block0:
+; vgbm %v24, 0
+; lrv %r3, 0(%r2)
+; vlvgf %v24, %r3, 3
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-logical.clif b/cranelift/filetests/filetests/isa/s390x/vec-logical.clif
new file mode 100644
index 000000000000..b0375f81dc10
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-logical.clif
@@ -0,0 +1,675 @@
+test compile precise-output
+target s390x
+
+function %vany_true_i64x2(i64x2) -> b1 {
+block0(v0: i64x2):
+ v1 = vany_true v0
+ return v1
+}
+
+; block0:
+; vgbm %v3, 0
+; vceqgs %v5, %v24, %v3
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_i32x4(i32x4) -> b1 {
+block0(v0: i32x4):
+ v1 = vany_true v0
+ return v1
+}
+
+; block0:
+; vgbm %v3, 0
+; vceqfs %v5, %v24, %v3
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_i16x8(i16x8) -> b1 {
+block0(v0: i16x8):
+ v1 = vany_true v0
+ return v1
+}
+
+; block0:
+; vgbm %v3, 0
+; vceqhs %v5, %v24, %v3
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_i8x16(i8x16) -> b1 {
+block0(v0: i8x16):
+ v1 = vany_true v0
+ return v1
+}
+
+; block0:
+; vgbm %v3, 0
+; vceqbs %v5, %v24, %v3
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vall_true_i64x2(i64x2) -> b1 {
+block0(v0: i64x2):
+ v1 = vall_true v0
+ return v1
+}
+
+; block0:
+; vgbm %v3, 0
+; vceqgs %v5, %v24, %v3
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_i32x4(i32x4) -> b1 {
+block0(v0: i32x4):
+ v1 = vall_true v0
+ return v1
+}
+
+; block0:
+; vgbm %v3, 0
+; vceqfs %v5, %v24, %v3
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_i16x8(i16x8) -> b1 {
+block0(v0: i16x8):
+ v1 = vall_true v0
+ return v1
+}
+
+; block0:
+; vgbm %v3, 0
+; vceqhs %v5, %v24, %v3
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_i8x16(i8x16) -> b1 {
+block0(v0: i8x16):
+ v1 = vall_true v0
+ return v1
+}
+
+; block0:
+; vgbm %v3, 0
+; vceqbs %v5, %v24, %v3
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vany_true_icmp_eq_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp eq v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vceqgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_icmp_ne_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp ne v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vceqgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_icmp_sgt_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp sgt v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vchgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_icmp_sle_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp sle v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vchgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_icmp_slt_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp slt v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vchgs %v5, %v25, %v24
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_icmp_sge_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp sge v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vchgs %v5, %v25, %v24
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_icmp_ugt_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp ugt v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vchlgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_icmp_ule_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp ule v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vchlgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_icmp_ult_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp ult v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vchlgs %v5, %v25, %v24
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_icmp_uge_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp uge v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vchlgs %v5, %v25, %v24
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_fcmp_eq_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp eq v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfcedbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_fcmp_ne_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ne v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfcedbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_fcmp_gt_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp gt v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfchdbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_fcmp_ule_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ule v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfchdbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_fcmp_ge_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ge v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfchedbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_fcmp_ult_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ult v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfchedbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_fcmp_lt_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp lt v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfchdbs %v5, %v25, %v24
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_fcmp_uge_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp uge v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfchdbs %v5, %v25, %v24
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vany_true_fcmp_le_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp le v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfchedbs %v5, %v25, %v24
+; lhi %r2, 0
+; lochino %r2, 1
+; br %r14
+
+function %vany_true_fcmp_ugt_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ugt v0, v1
+ v3 = vany_true v2
+ return v3
+}
+
+; block0:
+; vfchedbs %v5, %v25, %v24
+; lhi %r2, 0
+; lochine %r2, 1
+; br %r14
+
+function %vall_true_icmp_eq_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp eq v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vceqgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_icmp_ne_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp ne v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vceqgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_icmp_sgt_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp sgt v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vchgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_icmp_sle_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp sle v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vchgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_icmp_slt_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp slt v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vchgs %v5, %v25, %v24
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_icmp_sge_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp sge v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vchgs %v5, %v25, %v24
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_icmp_ugt_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp ugt v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vchlgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_icmp_ule_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp ule v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vchlgs %v5, %v24, %v25
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_icmp_ult_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp ult v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vchlgs %v5, %v25, %v24
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_icmp_uge_i64x2(i64x2, i64x2) -> b1 {
+block0(v0: i64x2, v1: i64x2):
+ v2 = icmp uge v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vchlgs %v5, %v25, %v24
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_fcmp_eq_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp eq v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfcedbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_fcmp_ne_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ne v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfcedbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_fcmp_gt_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp gt v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfchdbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_fcmp_ule_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ule v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfchdbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_fcmp_ge_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ge v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfchedbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_fcmp_ult_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ult v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfchedbs %v5, %v24, %v25
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_fcmp_lt_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp lt v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfchdbs %v5, %v25, %v24
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_fcmp_uge_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp uge v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfchdbs %v5, %v25, %v24
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vall_true_fcmp_le_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp le v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfchedbs %v5, %v25, %v24
+; lhi %r2, 0
+; lochie %r2, 1
+; br %r14
+
+function %vall_true_fcmp_ugt_f64x2(f64x2, f64x2) -> b1 {
+block0(v0: f64x2, v1: f64x2):
+ v2 = fcmp ugt v0, v1
+ v3 = vall_true v2
+ return v3
+}
+
+; block0:
+; vfchedbs %v5, %v25, %v24
+; lhi %r2, 0
+; lochio %r2, 1
+; br %r14
+
+function %vhigh_bits(i64x2) -> i64 {
+block0(v0: i64x2):
+ v1 = vhigh_bits.i64 v0
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x80808080808080808080808080800040 ; vl %v3, 0(%r1)
+; vbperm %v5, %v24, %v3
+; lgdr %r2, %f5
+; br %r14
+
+function %vhigh_bits(i32x4) -> i64 {
+block0(v0: i32x4):
+ v1 = vhigh_bits.i64 v0
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x80808080808080808080808000204060 ; vl %v3, 0(%r1)
+; vbperm %v5, %v24, %v3
+; lgdr %r2, %f5
+; br %r14
+
+function %vhigh_bits(i16x8) -> i64 {
+block0(v0: i16x8):
+ v1 = vhigh_bits.i64 v0
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x80808080808080800010203040506070 ; vl %v3, 0(%r1)
+; vbperm %v5, %v24, %v3
+; lgdr %r2, %f5
+; br %r14
+
+function %vhigh_bits(i8x16) -> i64 {
+block0(v0: i8x16):
+ v1 = vhigh_bits.i64 v0
+ return v1
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x00081018202830384048505860687078 ; vl %v3, 0(%r1)
+; vbperm %v5, %v24, %v3
+; lgdr %r2, %f5
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-permute.clif b/cranelift/filetests/filetests/isa/s390x/vec-permute.clif
new file mode 100644
index 000000000000..4e5f7019c5a4
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-permute.clif
@@ -0,0 +1,493 @@
+test compile precise-output
+target s390x
+
+function %swizzle(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = swizzle.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vgbm %v5, 0
+; vrepib %v7, 239
+; vno %v17, %v25, %v25
+; vmxlb %v19, %v7, %v17
+; vperm %v24, %v5, %v24, %v19
+; br %r14
+
+function %shuffle_0(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+ return v2
+}
+
+; block0:
+; vrepib %v5, 15
+; vperm %v24, %v24, %v25, %v5
+; br %r14
+
+function %shuffle_1(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [3 0 31 26 4 6 12 11 23 13 24 4 2 15 17 5]
+ return v2
+}
+
+; block0:
+; bras %r1, 20 ; data.u128 0x0a1e000d0b1702180403090b15100f0c ; vl %v5, 0(%r1)
+; vperm %v24, %v24, %v25, %v5
+; br %r14
+
+function %shuffle_2(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47]
+ return v2
+}
+
+; block0:
+; vgbm %v5, 1
+; bras %r1, 20 ; data.u128 0x8080808080808080808080808080800f ; vl %v7, 0(%r1)
+; vperm %v17, %v24, %v25, %v7
+; vn %v24, %v5, %v17
+; br %r14
+
+function %shuffle_vmrhg_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [24 25 26 27 28 29 30 31 8 9 10 11 12 13 14 15]
+ return v2
+}
+
+; block0:
+; vmrhg %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vmrhf_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [24 25 26 27 8 9 10 11 28 29 30 31 12 13 14 15]
+ return v2
+}
+
+; block0:
+; vmrhf %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vmrhh_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [24 25 8 9 26 27 10 11 28 29 12 13 30 31 14 15]
+ return v2
+}
+
+; block0:
+; vmrhh %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vmrhb_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [24 8 25 9 26 10 27 11 28 12 29 13 30 14 31 15]
+ return v2
+}
+
+; block0:
+; vmrhb %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vmrhg_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31]
+ return v2
+}
+
+; block0:
+; vmrhg %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vmrhf_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [8 9 10 11 24 25 26 27 12 13 14 15 28 29 30 31]
+ return v2
+}
+
+; block0:
+; vmrhf %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vmrhh_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [8 9 24 25 10 11 26 27 12 13 28 29 14 15 30 31]
+ return v2
+}
+
+; block0:
+; vmrhh %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vmrhb_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [8 24 9 25 10 26 11 27 12 28 13 29 14 30 15 31]
+ return v2
+}
+
+; block0:
+; vmrhb %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vmrhg_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15]
+ return v2
+}
+
+; block0:
+; vmrhg %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vmrhf_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [8 9 10 11 8 9 10 11 12 13 14 15 12 13 14 15]
+ return v2
+}
+
+; block0:
+; vmrhf %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vmrhh_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [8 9 8 9 10 11 10 11 12 13 12 13 14 15 14 15]
+ return v2
+}
+
+; block0:
+; vmrhh %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vmrhb_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [8 8 9 9 10 10 11 11 12 12 13 13 14 14 15 15]
+ return v2
+}
+
+; block0:
+; vmrhb %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vmrhg_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [24 25 26 27 28 29 30 31 24 25 26 27 28 29 30 31]
+ return v2
+}
+
+; block0:
+; vmrhg %v24, %v25, %v25
+; br %r14
+
+function %shuffle_vmrhf_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [24 25 26 27 24 25 26 27 28 29 30 31 28 29 30 31]
+ return v2
+}
+
+; block0:
+; vmrhf %v24, %v25, %v25
+; br %r14
+
+function %shuffle_vmrhh_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [24 25 24 25 26 27 26 27 28 29 28 29 30 31 30 31]
+ return v2
+}
+
+; block0:
+; vmrhh %v24, %v25, %v25
+; br %r14
+
+function %shuffle_vmrhb_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [24 24 25 25 26 26 27 27 28 28 29 29 30 30 31 31]
+ return v2
+}
+
+; block0:
+; vmrhb %v24, %v25, %v25
+; br %r14
+
+function %shuffle_vmrlg_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 18 19 20 21 22 23 0 1 2 3 4 5 6 7]
+ return v2
+}
+
+; block0:
+; vmrlg %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vmrlf_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 18 19 0 1 2 3 20 21 22 23 4 5 6 7]
+ return v2
+}
+
+; block0:
+; vmrlf %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vmrlh_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 0 1 18 19 2 3 20 21 4 5 22 23 6 7]
+ return v2
+}
+
+; block0:
+; vmrlh %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vmrlb_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 0 17 1 18 2 19 3 20 4 21 5 22 6 23 7]
+ return v2
+}
+
+; block0:
+; vmrlb %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vmrlg_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23]
+ return v2
+}
+
+; block0:
+; vmrlg %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vmrlf_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 2 3 16 17 18 19 4 5 6 7 20 21 22 23]
+ return v2
+}
+
+; block0:
+; vmrlf %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vmrlh_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 16 17 2 3 18 19 4 5 20 21 6 7 22 23]
+ return v2
+}
+
+; block0:
+; vmrlh %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vmrlb_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 16 1 17 2 18 3 19 4 20 5 21 6 22 7 23]
+ return v2
+}
+
+; block0:
+; vmrlb %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vmrlg_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7]
+ return v2
+}
+
+; block0:
+; vmrlg %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vmrlf_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 2 3 0 1 2 3 4 5 6 7 4 5 6 7]
+ return v2
+}
+
+; block0:
+; vmrlf %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vmrlh_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 0 1 2 3 2 3 4 5 4 5 6 7 6 7]
+ return v2
+}
+
+; block0:
+; vmrlh %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vmrlb_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7]
+ return v2
+}
+
+; block0:
+; vmrlb %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vmrlg_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 18 19 20 21 22 23 16 17 18 19 20 21 22 23]
+ return v2
+}
+
+; block0:
+; vmrlg %v24, %v25, %v25
+; br %r14
+
+function %shuffle_vmrlf_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 18 19 16 17 18 19 20 21 22 23 20 21 22 23]
+ return v2
+}
+
+; block0:
+; vmrlf %v24, %v25, %v25
+; br %r14
+
+function %shuffle_vmrlh_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 16 17 18 19 18 19 20 21 20 21 22 23 22 23]
+ return v2
+}
+
+; block0:
+; vmrlh %v24, %v25, %v25
+; br %r14
+
+function %shuffle_vmrlb_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 16 17 17 18 18 19 19 20 20 21 21 22 22 23 23]
+ return v2
+}
+
+; block0:
+; vmrlb %v24, %v25, %v25
+; br %r14
+
+;; Special patterns that can be implemented via PACK.
+function %shuffle_vpkg_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 18 19 24 25 26 27 0 1 2 3 8 9 10 11]
+ return v2
+}
+
+; block0:
+; vpkg %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vpkf_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 20 21 24 25 28 29 0 1 4 5 8 9 12 13]
+ return v2
+}
+
+; block0:
+; vpkf %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vpkh_xy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 18 20 22 24 26 28 30 0 2 4 6 8 10 12 14]
+ return v2
+}
+
+; block0:
+; vpkh %v24, %v24, %v25
+; br %r14
+
+function %shuffle_vpkg_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 2 3 8 9 10 11 16 17 18 19 24 25 26 27]
+ return v2
+}
+
+; block0:
+; vpkg %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vpkf_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 4 5 8 9 12 13 16 17 20 21 24 25 28 29]
+ return v2
+}
+
+; block0:
+; vpkf %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vpkh_yx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30]
+ return v2
+}
+
+; block0:
+; vpkh %v24, %v25, %v24
+; br %r14
+
+function %shuffle_vpkg_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 2 3 8 9 10 11 0 1 2 3 8 9 10 11]
+ return v2
+}
+
+; block0:
+; vpkg %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vpkf_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 1 4 5 8 9 12 13 0 1 4 5 8 9 12 13]
+ return v2
+}
+
+; block0:
+; vpkf %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vpkh_xx(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [0 2 4 6 8 10 12 14 0 2 4 6 8 10 12 14]
+ return v2
+}
+
+; block0:
+; vpkh %v24, %v24, %v24
+; br %r14
+
+function %shuffle_vpkg_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 18 19 24 25 26 27 16 17 18 19 24 25 26 27]
+ return v2
+}
+
+; block0:
+; vpkg %v24, %v25, %v25
+; br %r14
+
+function %shuffle_vpkf_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 17 20 21 24 25 28 29 16 17 20 21 24 25 28 29]
+ return v2
+}
+
+; block0:
+; vpkf %v24, %v25, %v25
+; br %r14
+
+function %shuffle_vpkh_yy(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle.i8x16 v0, v1, [16 18 20 22 24 26 28 30 16 18 20 22 24 26 28 30]
+ return v2
+}
+
+; block0:
+; vpkh %v24, %v25, %v25
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vec-shift-rotate.clif b/cranelift/filetests/filetests/isa/s390x/vec-shift-rotate.clif
new file mode 100644
index 000000000000..7713bd0f3340
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vec-shift-rotate.clif
@@ -0,0 +1,427 @@
+test compile precise-output
+target s390x
+
+function %rotr_i64x4_reg(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = rotr.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; lcr %r3, %r2
+; verllg %v24, %v24, 0(%r3)
+; br %r14
+
+function %rotr_i64x4_imm(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = iconst.i32 17
+ v2 = rotr.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; verllg %v24, %v24, 47
+; br %r14
+
+function %rotr_i32x4_reg(i32x4, i32) -> i32x4 {
+block0(v0: i32x4, v1: i32):
+ v2 = rotr.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; lcr %r3, %r2
+; verllf %v24, %v24, 0(%r3)
+; br %r14
+
+function %rotr_i32x4_imm(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = iconst.i32 17
+ v2 = rotr.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; verllf %v24, %v24, 15
+; br %r14
+
+function %rotr_i16x8_reg(i16x8, i16) -> i16x8 {
+block0(v0: i16x8, v1: i16):
+ v2 = rotr.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; lcr %r3, %r2
+; verllh %v24, %v24, 0(%r3)
+; br %r14
+
+function %rotr_i16x8_imm(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = iconst.i32 10
+ v2 = rotr.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; verllh %v24, %v24, 6
+; br %r14
+
+function %rotr_i8x16_reg(i8x16, i8) -> i8x16 {
+block0(v0: i8x16, v1: i8):
+ v2 = rotr.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; lcr %r3, %r2
+; verllb %v24, %v24, 0(%r3)
+; br %r14
+
+function %rotr_i8x16_imm(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = iconst.i32 3
+ v2 = rotr.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; verllb %v24, %v24, 5
+; br %r14
+
+function %rotl_i64x2_reg(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = rotl.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; verllg %v24, %v24, 0(%r2)
+; br %r14
+
+function %rotl_i64x2_imm(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = iconst.i32 17
+ v2 = rotl.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; verllg %v24, %v24, 17
+; br %r14
+
+function %rotl_i32x4_reg(i32x4, i32) -> i32x4 {
+block0(v0: i32x4, v1: i32):
+ v2 = rotl.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; verllf %v24, %v24, 0(%r2)
+; br %r14
+
+function %rotl_i32x4_imm(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = iconst.i32 17
+ v2 = rotl.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; verllf %v24, %v24, 17
+; br %r14
+
+function %rotl_i16x8_reg(i16x8, i16) -> i16x8 {
+block0(v0: i16x8, v1: i16):
+ v2 = rotl.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; verllh %v24, %v24, 0(%r2)
+; br %r14
+
+function %rotl_i16x8_imm(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = iconst.i32 10
+ v2 = rotl.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; verllh %v24, %v24, 10
+; br %r14
+
+function %rotl_i8x16_reg(i8x16, i8) -> i8x16 {
+block0(v0: i8x16, v1: i8):
+ v2 = rotl.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; verllb %v24, %v24, 0(%r2)
+; br %r14
+
+function %rotr_i8x16_imm(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = iconst.i32 3
+ v2 = rotl.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; verllb %v24, %v24, 3
+; br %r14
+
+function %ushr_i64x2_reg(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = ushr.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vesrlg %v24, %v24, 0(%r2)
+; br %r14
+
+function %ushr_i64x2_imm(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = iconst.i32 17
+ v2 = ushr.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vesrlg %v24, %v24, 17
+; br %r14
+
+function %ushr_i32x4_reg(i32x4, i32) -> i32x4 {
+block0(v0: i32x4, v1: i32):
+ v2 = ushr.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vesrlf %v24, %v24, 0(%r2)
+; br %r14
+
+function %ushr_i32x4_imm(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = iconst.i32 17
+ v2 = ushr.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vesrlf %v24, %v24, 17
+; br %r14
+
+function %ushr_i16x8_reg(i16x8, i16) -> i16x8 {
+block0(v0: i16x8, v1: i16):
+ v2 = ushr.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vesrlh %v24, %v24, 0(%r2)
+; br %r14
+
+function %ushr_i16x8_imm(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = iconst.i32 10
+ v2 = ushr.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vesrlh %v24, %v24, 10
+; br %r14
+
+function %ushr_i8x16_reg(i8x16, i8) -> i8x16 {
+block0(v0: i8x16, v1: i8):
+ v2 = ushr.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vesrlb %v24, %v24, 0(%r2)
+; br %r14
+
+function %ushr_i8x16_imm(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = iconst.i32 3
+ v2 = ushr.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vesrlb %v24, %v24, 3
+; br %r14
+
+function %ishl_i64x2_reg(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = ishl.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; veslg %v24, %v24, 0(%r2)
+; br %r14
+
+function %ishl_i64x2_imm(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = iconst.i32 17
+ v2 = ishl.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; veslg %v24, %v24, 17
+; br %r14
+
+function %ishl_i32x4_reg(i32x4, i32) -> i32x4 {
+block0(v0: i32x4, v1: i32):
+ v2 = ishl.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; veslf %v24, %v24, 0(%r2)
+; br %r14
+
+function %ishl_i32x4_imm(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = iconst.i32 17
+ v2 = ishl.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; veslf %v24, %v24, 17
+; br %r14
+
+function %ishl_i16x8_reg(i16x8, i16) -> i16x8 {
+block0(v0: i16x8, v1: i16):
+ v2 = ishl.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; veslh %v24, %v24, 0(%r2)
+; br %r14
+
+function %ishl_i16x8_imm(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = iconst.i32 10
+ v2 = ishl.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; veslh %v24, %v24, 10
+; br %r14
+
+function %ishl_i8x16_reg(i8x16, i8) -> i8x16 {
+block0(v0: i8x16, v1: i8):
+ v2 = ishl.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; veslb %v24, %v24, 0(%r2)
+; br %r14
+
+function %ishl_i8x16_imm(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = iconst.i32 3
+ v2 = ishl.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; veslb %v24, %v24, 3
+; br %r14
+
+function %sshr_i64x2_reg(i64x2, i64) -> i64x2 {
+block0(v0: i64x2, v1: i64):
+ v2 = sshr.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vesrag %v24, %v24, 0(%r2)
+; br %r14
+
+function %sshr_i64x2_imm(i64x2) -> i64x2 {
+block0(v0: i64x2):
+ v1 = iconst.i32 17
+ v2 = sshr.i64x2 v0, v1
+ return v2
+}
+
+; block0:
+; vesrag %v24, %v24, 17
+; br %r14
+
+function %sshr_i32x4_reg(i32x4, i32) -> i32x4 {
+block0(v0: i32x4, v1: i32):
+ v2 = sshr.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vesraf %v24, %v24, 0(%r2)
+; br %r14
+
+function %sshr_i32x4_imm(i32x4) -> i32x4 {
+block0(v0: i32x4):
+ v1 = iconst.i32 17
+ v2 = sshr.i32x4 v0, v1
+ return v2
+}
+
+; block0:
+; vesraf %v24, %v24, 17
+; br %r14
+
+function %sshr_i16x8_reg(i16x8, i16) -> i16x8 {
+block0(v0: i16x8, v1: i16):
+ v2 = sshr.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vesrah %v24, %v24, 0(%r2)
+; br %r14
+
+function %sshr_i16x8_imm(i16x8) -> i16x8 {
+block0(v0: i16x8):
+ v1 = iconst.i32 10
+ v2 = sshr.i16x8 v0, v1
+ return v2
+}
+
+; block0:
+; vesrah %v24, %v24, 10
+; br %r14
+
+function %sshr_i8x16_reg(i8x16, i8) -> i8x16 {
+block0(v0: i8x16, v1: i8):
+ v2 = sshr.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vesrab %v24, %v24, 0(%r2)
+; br %r14
+
+function %sshr_i8x16_imm(i8x16) -> i8x16 {
+block0(v0: i8x16):
+ v1 = iconst.i32 3
+ v2 = sshr.i8x16 v0, v1
+ return v2
+}
+
+; block0:
+; vesrab %v24, %v24, 3
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vecmem-arch13.clif b/cranelift/filetests/filetests/isa/s390x/vecmem-arch13.clif
new file mode 100644
index 000000000000..a60e7b619476
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vecmem-arch13.clif
@@ -0,0 +1,375 @@
+test compile precise-output
+target s390x arch13
+
+function %uload8x8_big(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = uload8x8 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuplhb %v24, %v3
+; br %r14
+
+function %uload16x4_big(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = uload16x4 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuplhh %v24, %v3
+; br %r14
+
+function %uload32x2_big(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = uload32x2 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuplhf %v24, %v3
+; br %r14
+
+function %sload8x8_big(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = sload8x8 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuphb %v24, %v3
+; br %r14
+
+function %sload16x4_big(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = sload16x4 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuphh %v24, %v3
+; br %r14
+
+function %sload32x2_big(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = sload32x2 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuphf %v24, %v3
+; br %r14
+
+function %load_i8x16_big(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8x16 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_i16x8_big(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16x8 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_i32x4_big(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32x4 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_i64x2_big(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64x2 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_f32x4_big(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32x4 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_f64x2_big(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64x2 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %store_i8x16_big(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ store.i8x16 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_i16x8_big(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ store.i16x8 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_i32x4_big(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ store.i32x4 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_i64x2_big(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ store.i64x2 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_f32x4_big(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ store.f32x4 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_f64x2_big(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ store.f64x2 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %uload8x8_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = uload8x8 little v0
+ return v1
+}
+
+; block0:
+; vlebrg %v3, 0(%r2), 0
+; vuplhb %v24, %v3
+; br %r14
+
+function %uload16x4_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = uload16x4 little v0
+ return v1
+}
+
+; block0:
+; vlebrg %v3, 0(%r2), 0
+; vuplhh %v24, %v3
+; br %r14
+
+function %uload32x2_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = uload32x2 little v0
+ return v1
+}
+
+; block0:
+; vlebrg %v3, 0(%r2), 0
+; vuplhf %v24, %v3
+; br %r14
+
+function %sload8x8_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = sload8x8 little v0
+ return v1
+}
+
+; block0:
+; vlebrg %v3, 0(%r2), 0
+; vuphb %v24, %v3
+; br %r14
+
+function %sload16x4_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = sload16x4 little v0
+ return v1
+}
+
+; block0:
+; vlebrg %v3, 0(%r2), 0
+; vuphh %v24, %v3
+; br %r14
+
+function %sload32x2_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = sload32x2 little v0
+ return v1
+}
+
+; block0:
+; vlebrg %v3, 0(%r2), 0
+; vuphf %v24, %v3
+; br %r14
+
+function %load_i8x16_little(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8x16 little v0
+ return v1
+}
+
+; block0:
+; vlbrq %v24, 0(%r2)
+; br %r14
+
+function %load_i16x8_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16x8 little v0
+ return v1
+}
+
+; block0:
+; vlbrq %v24, 0(%r2)
+; br %r14
+
+function %load_i32x4_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32x4 little v0
+ return v1
+}
+
+; block0:
+; vlbrq %v24, 0(%r2)
+; br %r14
+
+function %load_i64x2_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64x2 little v0
+ return v1
+}
+
+; block0:
+; vlbrq %v24, 0(%r2)
+; br %r14
+
+function %load_f32x4_little(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32x4 little v0
+ return v1
+}
+
+; block0:
+; vlbrq %v24, 0(%r2)
+; br %r14
+
+function %load_f64x2_little(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64x2 little v0
+ return v1
+}
+
+; block0:
+; vlbrq %v24, 0(%r2)
+; br %r14
+
+function %store_i8x16_little(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ store.i8x16 little v0, v1
+ return
+}
+
+; block0:
+; vstbrq %v24, 0(%r2)
+; br %r14
+
+function %store_i16x8_little(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ store.i16x8 little v0, v1
+ return
+}
+
+; block0:
+; vstbrq %v24, 0(%r2)
+; br %r14
+
+function %store_i32x4_little(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ store.i32x4 little v0, v1
+ return
+}
+
+; block0:
+; vstbrq %v24, 0(%r2)
+; br %r14
+
+function %store_i64x2_little(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ store.i64x2 little v0, v1
+ return
+}
+
+; block0:
+; vstbrq %v24, 0(%r2)
+; br %r14
+
+function %store_f32x4_little(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ store.f32x4 little v0, v1
+ return
+}
+
+; block0:
+; vstbrq %v24, 0(%r2)
+; br %r14
+
+function %store_f64x2_little(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ store.f64x2 little v0, v1
+ return
+}
+
+; block0:
+; vstbrq %v24, 0(%r2)
+; br %r14
+
diff --git a/cranelift/filetests/filetests/isa/s390x/vecmem.clif b/cranelift/filetests/filetests/isa/s390x/vecmem.clif
new file mode 100644
index 000000000000..5cb297dde83b
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/vecmem.clif
@@ -0,0 +1,463 @@
+test compile precise-output
+target s390x
+
+function %uload8x8_big(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = uload8x8 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuplhb %v24, %v3
+; br %r14
+
+function %uload16x4_big(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = uload16x4 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuplhh %v24, %v3
+; br %r14
+
+function %uload32x2_big(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = uload32x2 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuplhf %v24, %v3
+; br %r14
+
+function %sload8x8_big(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = sload8x8 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuphb %v24, %v3
+; br %r14
+
+function %sload16x4_big(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = sload16x4 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuphh %v24, %v3
+; br %r14
+
+function %sload32x2_big(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = sload32x2 big v0
+ return v1
+}
+
+; block0:
+; ld %f3, 0(%r2)
+; vuphf %v24, %v3
+; br %r14
+
+function %load_i8x16_big(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8x16 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_i16x8_big(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16x8 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_i32x4_big(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32x4 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_i64x2_big(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64x2 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_f32x4_big(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32x4 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %load_f64x2_big(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64x2 big v0
+ return v1
+}
+
+; block0:
+; vl %v24, 0(%r2)
+; br %r14
+
+function %store_i8x16_big(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ store.i8x16 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_i16x8_big(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ store.i16x8 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_i32x4_big(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ store.i32x4 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_i64x2_big(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ store.i64x2 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_f32x4_big(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ store.f32x4 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %store_f64x2_big(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ store.f64x2 big v0, v1
+ return
+}
+
+; block0:
+; vst %v24, 0(%r2)
+; br %r14
+
+function %uload8x8_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = uload8x8 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; ldgr %f5, %r5
+; vuplhb %v24, %v5
+; br %r14
+
+function %uload16x4_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = uload16x4 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; ldgr %f5, %r5
+; vuplhh %v24, %v5
+; br %r14
+
+function %uload32x2_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = uload32x2 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; ldgr %f5, %r5
+; vuplhf %v24, %v5
+; br %r14
+
+function %sload8x8_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = sload8x8 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; ldgr %f5, %r5
+; vuphb %v24, %v5
+; br %r14
+
+function %sload16x4_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = sload16x4 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; ldgr %f5, %r5
+; vuphh %v24, %v5
+; br %r14
+
+function %sload32x2_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = sload32x2 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; ldgr %f5, %r5
+; vuphf %v24, %v5
+; br %r14
+
+function %load_i8x16_little(i64) -> i8x16 {
+block0(v0: i64):
+ v1 = load.i8x16 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; lrvg %r3, 8(%r2)
+; vlvgp %v24, %r3, %r5
+; br %r14
+
+function %load_i16x8_little(i64) -> i16x8 {
+block0(v0: i64):
+ v1 = load.i16x8 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; lrvg %r3, 8(%r2)
+; vlvgp %v24, %r3, %r5
+; br %r14
+
+function %load_i32x4_little(i64) -> i32x4 {
+block0(v0: i64):
+ v1 = load.i32x4 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; lrvg %r3, 8(%r2)
+; vlvgp %v24, %r3, %r5
+; br %r14
+
+function %load_i64x2_little(i64) -> i64x2 {
+block0(v0: i64):
+ v1 = load.i64x2 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; lrvg %r3, 8(%r2)
+; vlvgp %v24, %r3, %r5
+; br %r14
+
+function %load_f32x4_little(i64) -> f32x4 {
+block0(v0: i64):
+ v1 = load.f32x4 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; lrvg %r3, 8(%r2)
+; vlvgp %v24, %r3, %r5
+; br %r14
+
+function %load_f64x2_little(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64x2 little v0
+ return v1
+}
+
+; block0:
+; lrvg %r5, 0(%r2)
+; lrvg %r3, 8(%r2)
+; vlvgp %v24, %r3, %r5
+; br %r14
+
+function %load_f64x2_sum_little(i64, i64) -> f64x2 {
+block0(v0: i64, v1: i64):
+ v2 = iadd.i64 v0, v1
+ v3 = load.f64x2 little v2
+ return v3
+}
+
+; block0:
+; lrvg %r4, 0(%r3,%r2)
+; lrvg %r5, 8(%r3,%r2)
+; vlvgp %v24, %r5, %r4
+; br %r14
+
+function %load_f64x2_off_little(i64) -> f64x2 {
+block0(v0: i64):
+ v1 = load.f64x2 little v0+128
+ return v1
+}
+
+; block0:
+; lrvg %r5, 128(%r2)
+; lrvg %r3, 136(%r2)
+; vlvgp %v24, %r3, %r5
+; br %r14
+
+function %store_i8x16_little(i8x16, i64) {
+block0(v0: i8x16, v1: i64):
+ store.i8x16 little v0, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 1
+; vlgvg %r4, %v24, 0
+; strvg %r3, 0(%r2)
+; strvg %r4, 8(%r2)
+; br %r14
+
+function %store_i16x8_little(i16x8, i64) {
+block0(v0: i16x8, v1: i64):
+ store.i16x8 little v0, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 1
+; vlgvg %r4, %v24, 0
+; strvg %r3, 0(%r2)
+; strvg %r4, 8(%r2)
+; br %r14
+
+function %store_i32x4_little(i32x4, i64) {
+block0(v0: i32x4, v1: i64):
+ store.i32x4 little v0, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 1
+; vlgvg %r4, %v24, 0
+; strvg %r3, 0(%r2)
+; strvg %r4, 8(%r2)
+; br %r14
+
+function %store_i64x2_little(i64x2, i64) {
+block0(v0: i64x2, v1: i64):
+ store.i64x2 little v0, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 1
+; vlgvg %r4, %v24, 0
+; strvg %r3, 0(%r2)
+; strvg %r4, 8(%r2)
+; br %r14
+
+function %store_f32x4_little(f32x4, i64) {
+block0(v0: f32x4, v1: i64):
+ store.f32x4 little v0, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 1
+; vlgvg %r4, %v24, 0
+; strvg %r3, 0(%r2)
+; strvg %r4, 8(%r2)
+; br %r14
+
+function %store_f64x2_little(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ store.f64x2 little v0, v1
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 1
+; vlgvg %r4, %v24, 0
+; strvg %r3, 0(%r2)
+; strvg %r4, 8(%r2)
+; br %r14
+
+function %store_f64x2_sum_little(f64x2, i64, i64) {
+block0(v0: f64x2, v1: i64, v2: i64):
+ v3 = iadd.i64 v1, v2
+ store.f64x2 little v0, v3
+ return
+}
+
+; block0:
+; vlgvg %r5, %v24, 1
+; vlgvg %r4, %v24, 0
+; strvg %r5, 0(%r3,%r2)
+; strvg %r4, 8(%r3,%r2)
+; br %r14
+
+function %store_f64x2_off_little(f64x2, i64) {
+block0(v0: f64x2, v1: i64):
+ store.f64x2 little v0, v1+128
+ return
+}
+
+; block0:
+; vlgvg %r3, %v24, 1
+; vlgvg %r4, %v24, 0
+; strvg %r3, 128(%r2)
+; strvg %r4, 136(%r2)
+; br %r14
+
diff --git a/cranelift/filetests/filetests/runtests/fmax-pseudo.clif b/cranelift/filetests/filetests/runtests/fmax-pseudo.clif
index acd7b290dd8c..f5bf2a002ad1 100644
--- a/cranelift/filetests/filetests/runtests/fmax-pseudo.clif
+++ b/cranelift/filetests/filetests/runtests/fmax-pseudo.clif
@@ -2,6 +2,7 @@ test interpret
test run
target x86_64
target aarch64
+; target s390x FIXME: This currently fails under qemu due to a qemu bug
function %fmax_p_f32(f32, f32) -> f32 {
block0(v0: f32, v1: f32):
diff --git a/cranelift/filetests/filetests/runtests/fmin-max-pseudo-vector.clif b/cranelift/filetests/filetests/runtests/fmin-max-pseudo-vector.clif
index 9bbba57559ab..520f3aaff85b 100644
--- a/cranelift/filetests/filetests/runtests/fmin-max-pseudo-vector.clif
+++ b/cranelift/filetests/filetests/runtests/fmin-max-pseudo-vector.clif
@@ -1,7 +1,7 @@
test run
-; target s390x TODO: Not yet implemented on s390x
set enable_simd
target aarch64
+; target s390x FIXME: This currently fails under qemu due to a qemu bug
target x86_64 skylake
function %fmin_pseudo_f32x4(f32x4, f32x4) -> f32x4 {
diff --git a/cranelift/filetests/filetests/runtests/fmin-pseudo.clif b/cranelift/filetests/filetests/runtests/fmin-pseudo.clif
index fc88e34611f1..cb4857d8daba 100644
--- a/cranelift/filetests/filetests/runtests/fmin-pseudo.clif
+++ b/cranelift/filetests/filetests/runtests/fmin-pseudo.clif
@@ -2,6 +2,7 @@ test interpret
test run
target x86_64
target aarch64
+; target s390x FIXME: This currently fails under qemu due to a qemu bug
function %fmin_p_f32(f32, f32) -> f32 {
block0(v0: f32, v1: f32):
diff --git a/cranelift/filetests/filetests/runtests/shifts-small-types.clif b/cranelift/filetests/filetests/runtests/shifts-small-types.clif
index eae20a8ef01e..9b2207a3933d 100644
--- a/cranelift/filetests/filetests/runtests/shifts-small-types.clif
+++ b/cranelift/filetests/filetests/runtests/shifts-small-types.clif
@@ -1,7 +1,8 @@
test run
target aarch64
+target s390x
-; TODO: Merge this with the main shifts file when x86_64 & s390x passes these.
+; TODO: Merge this with the main shifts file when x86_64 passes these.
function %ishl_i16_i64(i16, i64) -> i16 {
block0(v0: i16, v1: i64):
diff --git a/cranelift/filetests/filetests/runtests/simd-arithmetic-nondeterministic-aarch64.clif b/cranelift/filetests/filetests/runtests/simd-arithmetic-nondeterministic-aarch64.clif
index f657d1e5336e..92ffddeef20f 100644
--- a/cranelift/filetests/filetests/runtests/simd-arithmetic-nondeterministic-aarch64.clif
+++ b/cranelift/filetests/filetests/runtests/simd-arithmetic-nondeterministic-aarch64.clif
@@ -3,6 +3,7 @@
; simd-arithmetic-nondeterministic*.clif as well.
test run
target aarch64
+target s390x
function %fmax_f64x2(f64x2, f64x2) -> f64x2 {
block0(v0: f64x2, v1: f64x2):
diff --git a/cranelift/filetests/filetests/runtests/simd-arithmetic.clif b/cranelift/filetests/filetests/runtests/simd-arithmetic.clif
index 1ca8e8fcfa3b..58a0dc1c21f6 100644
--- a/cranelift/filetests/filetests/runtests/simd-arithmetic.clif
+++ b/cranelift/filetests/filetests/runtests/simd-arithmetic.clif
@@ -1,6 +1,6 @@
test run
target aarch64
-; target s390x TODO: Not yet implemented on s390x
+target s390x
set enable_simd
target x86_64 skylake
diff --git a/cranelift/filetests/filetests/runtests/simd-bitselect-to-vselect.clif b/cranelift/filetests/filetests/runtests/simd-bitselect-to-vselect.clif
index a2086b0426b7..4021e89fee42 100644
--- a/cranelift/filetests/filetests/runtests/simd-bitselect-to-vselect.clif
+++ b/cranelift/filetests/filetests/runtests/simd-bitselect-to-vselect.clif
@@ -1,6 +1,6 @@
test run
target aarch64
-; target s390x TODO: Not yet implemented on s390x
+target s390x
set opt_level=speed_and_size
set enable_simd
target x86_64 skylake
diff --git a/cranelift/filetests/filetests/runtests/simd-bitselect.clif b/cranelift/filetests/filetests/runtests/simd-bitselect.clif
index 3d67ff290504..18027373f8c8 100644
--- a/cranelift/filetests/filetests/runtests/simd-bitselect.clif
+++ b/cranelift/filetests/filetests/runtests/simd-bitselect.clif
@@ -1,6 +1,7 @@
test run
set enable_simd
target aarch64
+target s390x
target x86_64 has_sse3 has_ssse3 has_sse41
function %bitselect_i32x4(i32x4, i32x4, i32x4) -> i32x4 {
diff --git a/cranelift/filetests/filetests/runtests/simd-bitwise-run.clif b/cranelift/filetests/filetests/runtests/simd-bitwise-run.clif
index af7b24d5e64a..ce3ffa5321df 100644
--- a/cranelift/filetests/filetests/runtests/simd-bitwise-run.clif
+++ b/cranelift/filetests/filetests/runtests/simd-bitwise-run.clif
@@ -1,6 +1,7 @@
test run
set enable_simd
target aarch64
+; target s390x FIXME: s390x implements modulo semantics for shift counts
target x86_64 skylake
; TODO: once available, replace all lane extraction with `icmp + all_ones`
diff --git a/cranelift/filetests/filetests/runtests/simd-bitwise.clif b/cranelift/filetests/filetests/runtests/simd-bitwise.clif
index 670844db22bf..251f9516c15d 100644
--- a/cranelift/filetests/filetests/runtests/simd-bitwise.clif
+++ b/cranelift/filetests/filetests/runtests/simd-bitwise.clif
@@ -1,6 +1,6 @@
test run
target aarch64
-; target s390x TODO: Not yet implemented on s390x
+; target s390x FIXME: s390x implements modulo semantics for shift counts
set enable_simd
target x86_64 skylake
diff --git a/cranelift/filetests/filetests/runtests/simd-comparison.clif b/cranelift/filetests/filetests/runtests/simd-comparison.clif
index cd8341127d3f..dd8c6a80b2ef 100644
--- a/cranelift/filetests/filetests/runtests/simd-comparison.clif
+++ b/cranelift/filetests/filetests/runtests/simd-comparison.clif
@@ -1,6 +1,6 @@
test run
target aarch64
-; target s390x TODO: Not yet implemented on s390x
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-conversion.clif b/cranelift/filetests/filetests/runtests/simd-conversion.clif
index 2903a34e1680..da8bced238af 100644
--- a/cranelift/filetests/filetests/runtests/simd-conversion.clif
+++ b/cranelift/filetests/filetests/runtests/simd-conversion.clif
@@ -1,6 +1,6 @@
test run
target aarch64
-; target s390x TODO: Not yet implemented on s390x
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-extractlane.clif b/cranelift/filetests/filetests/runtests/simd-extractlane.clif
index 6aba1c67ba61..471130f8c252 100644
--- a/cranelift/filetests/filetests/runtests/simd-extractlane.clif
+++ b/cranelift/filetests/filetests/runtests/simd-extractlane.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-iabs.clif b/cranelift/filetests/filetests/runtests/simd-iabs.clif
index 95b53844c624..b9d6468f9c51 100644
--- a/cranelift/filetests/filetests/runtests/simd-iabs.clif
+++ b/cranelift/filetests/filetests/runtests/simd-iabs.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-iaddpairwise.clif b/cranelift/filetests/filetests/runtests/simd-iaddpairwise.clif
index 92f5d776fe6d..3deeb6cddd20 100644
--- a/cranelift/filetests/filetests/runtests/simd-iaddpairwise.clif
+++ b/cranelift/filetests/filetests/runtests/simd-iaddpairwise.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
function %iaddp_i8x16(i8x16, i8x16) -> i8x16 {
block0(v0: i8x16, v1: i8x16):
diff --git a/cranelift/filetests/filetests/runtests/simd-insertlane.clif b/cranelift/filetests/filetests/runtests/simd-insertlane.clif
index f3624f1bf19f..e0965d8324e3 100644
--- a/cranelift/filetests/filetests/runtests/simd-insertlane.clif
+++ b/cranelift/filetests/filetests/runtests/simd-insertlane.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
@@ -30,4 +31,4 @@ block0(v0: i64x2, v1: i64):
v2 = insertlane v0, v1, 0
return v2
}
-; run: %insertlane_0([1 1], 5000000000) == [5000000000 1]
\ No newline at end of file
+; run: %insertlane_0([1 1], 5000000000) == [5000000000 1]
diff --git a/cranelift/filetests/filetests/runtests/simd-lane-access.clif b/cranelift/filetests/filetests/runtests/simd-lane-access.clif
index 0818bdd85b89..d43a0e20cf63 100644
--- a/cranelift/filetests/filetests/runtests/simd-lane-access.clif
+++ b/cranelift/filetests/filetests/runtests/simd-lane-access.clif
@@ -1,6 +1,6 @@
test run
target aarch64
-; target s390x TODO: Not yet implemented on s390x
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-logical.clif b/cranelift/filetests/filetests/runtests/simd-logical.clif
index 081d4892de81..406ea9698ddd 100644
--- a/cranelift/filetests/filetests/runtests/simd-logical.clif
+++ b/cranelift/filetests/filetests/runtests/simd-logical.clif
@@ -1,6 +1,6 @@
test run
target aarch64
-; target s390x TODO: Not yet implemented on s390x
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-min-max.clif b/cranelift/filetests/filetests/runtests/simd-min-max.clif
index 54653616ffbf..7a4cc0a0784d 100644
--- a/cranelift/filetests/filetests/runtests/simd-min-max.clif
+++ b/cranelift/filetests/filetests/runtests/simd-min-max.clif
@@ -1,6 +1,7 @@
test run
target aarch64
target x86_64
+target s390x
function %imin_i8x16(i8x16, i8x16) -> i8x16 {
block0(v0: i8x16, v1: i8x16):
diff --git a/cranelift/filetests/filetests/runtests/simd-saddsat.clif b/cranelift/filetests/filetests/runtests/simd-saddsat.clif
index 515cc83a4a2e..104041204932 100644
--- a/cranelift/filetests/filetests/runtests/simd-saddsat.clif
+++ b/cranelift/filetests/filetests/runtests/simd-saddsat.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-shuffle.clif b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
index b7850a578165..eaabb23768cf 100644
--- a/cranelift/filetests/filetests/runtests/simd-shuffle.clif
+++ b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-snarrow.clif b/cranelift/filetests/filetests/runtests/simd-snarrow.clif
index 082e86c179f2..86d3ee2100bc 100644
--- a/cranelift/filetests/filetests/runtests/simd-snarrow.clif
+++ b/cranelift/filetests/filetests/runtests/simd-snarrow.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-splat.clif b/cranelift/filetests/filetests/runtests/simd-splat.clif
index 19892cb29b7c..1cfef52c78b4 100644
--- a/cranelift/filetests/filetests/runtests/simd-splat.clif
+++ b/cranelift/filetests/filetests/runtests/simd-splat.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-sqmulroundsat-aarch64.clif b/cranelift/filetests/filetests/runtests/simd-sqmulroundsat-aarch64.clif
index f6809ddc5c80..91554360b664 100644
--- a/cranelift/filetests/filetests/runtests/simd-sqmulroundsat-aarch64.clif
+++ b/cranelift/filetests/filetests/runtests/simd-sqmulroundsat-aarch64.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
;; x86_64 hasn't implemented this for `i32x4`
function %sqmulrs_i32x4(i32x4, i32x4) -> i32x4 {
diff --git a/cranelift/filetests/filetests/runtests/simd-sqmulroundsat.clif b/cranelift/filetests/filetests/runtests/simd-sqmulroundsat.clif
index 723696d25a8f..d7d3ffec7b28 100644
--- a/cranelift/filetests/filetests/runtests/simd-sqmulroundsat.clif
+++ b/cranelift/filetests/filetests/runtests/simd-sqmulroundsat.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-ssubsat.clif b/cranelift/filetests/filetests/runtests/simd-ssubsat.clif
index 8841f2275f66..1a517b483a8e 100644
--- a/cranelift/filetests/filetests/runtests/simd-ssubsat.clif
+++ b/cranelift/filetests/filetests/runtests/simd-ssubsat.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-swidenhigh.clif b/cranelift/filetests/filetests/runtests/simd-swidenhigh.clif
index 1d9c15581b76..169c9122e376 100644
--- a/cranelift/filetests/filetests/runtests/simd-swidenhigh.clif
+++ b/cranelift/filetests/filetests/runtests/simd-swidenhigh.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-swidenlow.clif b/cranelift/filetests/filetests/runtests/simd-swidenlow.clif
index bee577072d13..6c014ad4a4d0 100644
--- a/cranelift/filetests/filetests/runtests/simd-swidenlow.clif
+++ b/cranelift/filetests/filetests/runtests/simd-swidenlow.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-swizzle.clif b/cranelift/filetests/filetests/runtests/simd-swizzle.clif
index 390780879c77..e1c7fba879da 100644
--- a/cranelift/filetests/filetests/runtests/simd-swizzle.clif
+++ b/cranelift/filetests/filetests/runtests/simd-swizzle.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-uaddsat.clif b/cranelift/filetests/filetests/runtests/simd-uaddsat.clif
index d0af940abdee..5610d4d36c3a 100644
--- a/cranelift/filetests/filetests/runtests/simd-uaddsat.clif
+++ b/cranelift/filetests/filetests/runtests/simd-uaddsat.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-unarrow.clif b/cranelift/filetests/filetests/runtests/simd-unarrow.clif
index e535df5e0778..f15a3217570c 100644
--- a/cranelift/filetests/filetests/runtests/simd-unarrow.clif
+++ b/cranelift/filetests/filetests/runtests/simd-unarrow.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-usubsat.clif b/cranelift/filetests/filetests/runtests/simd-usubsat.clif
index ca8747c3e900..55a85c8c895b 100644
--- a/cranelift/filetests/filetests/runtests/simd-usubsat.clif
+++ b/cranelift/filetests/filetests/runtests/simd-usubsat.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-uunarrow.clif b/cranelift/filetests/filetests/runtests/simd-uunarrow.clif
index b2a68c44802f..11ff104db149 100644
--- a/cranelift/filetests/filetests/runtests/simd-uunarrow.clif
+++ b/cranelift/filetests/filetests/runtests/simd-uunarrow.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
; x86_64 panics: `Did not match fcvt input!
; thread 'worker #0' panicked at 'register allocation: Analysis(EntryLiveinValues([v2V]))', cranelift/codegen/src/machinst/compile.rs:96:10`
diff --git a/cranelift/filetests/filetests/runtests/simd-uwidenhigh.clif b/cranelift/filetests/filetests/runtests/simd-uwidenhigh.clif
index 959b6acd7336..aaf8d4102f99 100644
--- a/cranelift/filetests/filetests/runtests/simd-uwidenhigh.clif
+++ b/cranelift/filetests/filetests/runtests/simd-uwidenhigh.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-uwidenlow.clif b/cranelift/filetests/filetests/runtests/simd-uwidenlow.clif
index fab64406b946..90f14bb1d331 100644
--- a/cranelift/filetests/filetests/runtests/simd-uwidenlow.clif
+++ b/cranelift/filetests/filetests/runtests/simd-uwidenlow.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-valltrue.clif b/cranelift/filetests/filetests/runtests/simd-valltrue.clif
index c39a2702e6a4..c799893ac8e2 100644
--- a/cranelift/filetests/filetests/runtests/simd-valltrue.clif
+++ b/cranelift/filetests/filetests/runtests/simd-valltrue.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
target x86_64
function %vall_true_b8x16(b8x16) -> b1 {
diff --git a/cranelift/filetests/filetests/runtests/simd-vanytrue.clif b/cranelift/filetests/filetests/runtests/simd-vanytrue.clif
index 74b99d785e4c..28e1c60a7d50 100644
--- a/cranelift/filetests/filetests/runtests/simd-vanytrue.clif
+++ b/cranelift/filetests/filetests/runtests/simd-vanytrue.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
target x86_64
function %vany_true_b8x16(b8x16) -> b1 {
diff --git a/cranelift/filetests/filetests/runtests/simd-vconst.clif b/cranelift/filetests/filetests/runtests/simd-vconst.clif
index b2398b6ec01a..5aa5386484f4 100644
--- a/cranelift/filetests/filetests/runtests/simd-vconst.clif
+++ b/cranelift/filetests/filetests/runtests/simd-vconst.clif
@@ -1,5 +1,5 @@
test run
-; target s390x TODO: Not yet implemented on s390x
+target s390x
target aarch64
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-vhighbits.clif b/cranelift/filetests/filetests/runtests/simd-vhighbits.clif
index e4ed0e42cf7e..d22abb702414 100644
--- a/cranelift/filetests/filetests/runtests/simd-vhighbits.clif
+++ b/cranelift/filetests/filetests/runtests/simd-vhighbits.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-vselect.clif b/cranelift/filetests/filetests/runtests/simd-vselect.clif
index 53ef6f6353d7..db5f9180433e 100644
--- a/cranelift/filetests/filetests/runtests/simd-vselect.clif
+++ b/cranelift/filetests/filetests/runtests/simd-vselect.clif
@@ -1,6 +1,6 @@
test interpret
test run
-; target s390x TODO: Not yet implemented on s390x
+target s390x
target aarch64
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd-wideningpairwisedotproducts.clif b/cranelift/filetests/filetests/runtests/simd-wideningpairwisedotproducts.clif
index dcfaba0294a9..c38099c429f0 100644
--- a/cranelift/filetests/filetests/runtests/simd-wideningpairwisedotproducts.clif
+++ b/cranelift/filetests/filetests/runtests/simd-wideningpairwisedotproducts.clif
@@ -1,6 +1,7 @@
test interpret
test run
target aarch64
+target s390x
set enable_simd
target x86_64 has_sse3 has_ssse3 has_sse41
diff --git a/cranelift/filetests/filetests/runtests/simd_compare_zero.clif b/cranelift/filetests/filetests/runtests/simd_compare_zero.clif
index d3a8c655d171..445ccbcc148b 100644
--- a/cranelift/filetests/filetests/runtests/simd_compare_zero.clif
+++ b/cranelift/filetests/filetests/runtests/simd_compare_zero.clif
@@ -1,5 +1,6 @@
test run
target aarch64
+target s390x
; raw_bitcast is needed to get around issue with "bint" on aarch64