From c6b57ed444d4fe8d1a8b680696c9ce74da747906 Mon Sep 17 00:00:00 2001 From: adhoc Date: Thu, 16 Jan 2025 00:56:43 +0100 Subject: [PATCH 1/2] implement x64 xchg --- .../x64/atomic/rmw/i32_atomic_rmw16_xchgu.wat | 35 +++++++++ .../x64/atomic/rmw/i32_atomic_rmw8_xchgu.wat | 30 ++++++++ .../x64/atomic/rmw/i32_atomic_rmw_xchg.wat | 34 +++++++++ .../x64/atomic/rmw/i64_atomic_rmw16_xchgu.wat | 35 +++++++++ .../x64/atomic/rmw/i64_atomic_rmw32_addu.wat | 15 ++-- .../x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat | 35 +++++++++ .../x64/atomic/rmw/i64_atomic_rmw8_xchgu.wat | 30 ++++++++ .../x64/atomic/rmw/i64_atomic_rmw_xchg.wat | 34 +++++++++ winch/codegen/src/isa/x64/asm.rs | 25 ++++++ winch/codegen/src/isa/x64/masm.rs | 3 + winch/codegen/src/masm.rs | 1 + winch/codegen/src/visitor.rs | 76 +++++++++++++++++-- 12 files changed, 340 insertions(+), 13 deletions(-) create mode 100644 tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw16_xchgu.wat create mode 100644 tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw8_xchgu.wat create mode 100644 tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw_xchg.wat create mode 100644 tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw16_xchgu.wat create mode 100644 tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat create mode 100644 tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw8_xchgu.wat create mode 100644 tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw_xchg.wat diff --git a/tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw16_xchgu.wat b/tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw16_xchgu.wat new file mode 100644 index 000000000000..eb183c017314 --- /dev/null +++ b/tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw16_xchgu.wat @@ -0,0 +1,35 @@ +;;! target = "x86_64" +;;! test = "winch" + +(module + (memory 1 1 shared) + (func (export "_start") (result i32) + (i32.atomic.rmw16.xchg_u (i32.const 0) (i32.const 42)))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x10(%r11), %r11 +;; addq $0x10, %r11 +;; cmpq %rsp, %r11 +;; ja 0x5f +;; 1c: movq %rdi, %r14 +;; subq $0x10, %rsp +;; movq %rdi, 8(%rsp) +;; movq %rsi, (%rsp) +;; movl $0x2a, %eax +;; movl $0, %ecx +;; andw $1, %cx +;; cmpw $0, %cx +;; jne 0x61 +;; 44: movl $0, %ecx +;; movq 0x58(%r14), %r11 +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; xchgw %ax, (%rdx) +;; movzwl %ax, %eax +;; addq $0x10, %rsp +;; popq %rbp +;; retq +;; 5f: ud2 +;; 61: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw8_xchgu.wat b/tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw8_xchgu.wat new file mode 100644 index 000000000000..7d63f9633b9d --- /dev/null +++ b/tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw8_xchgu.wat @@ -0,0 +1,30 @@ +;;! target = "x86_64" +;;! test = "winch" + +(module + (memory 1 1 shared) + (func (export "_start") (result i32) + (i32.atomic.rmw8.xchg_u (i32.const 0) (i32.const 42)))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x10(%r11), %r11 +;; addq $0x10, %r11 +;; cmpq %rsp, %r11 +;; ja 0x4b +;; 1c: movq %rdi, %r14 +;; subq $0x10, %rsp +;; movq %rdi, 8(%rsp) +;; movq %rsi, (%rsp) +;; movl $0x2a, %eax +;; movl $0, %ecx +;; movq 0x58(%r14), %r11 +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; xchgb %al, (%rdx) +;; movzbl %al, %eax +;; addq $0x10, %rsp +;; popq %rbp +;; retq +;; 4b: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw_xchg.wat b/tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw_xchg.wat new file mode 100644 index 000000000000..60990feae396 --- /dev/null +++ b/tests/disas/winch/x64/atomic/rmw/i32_atomic_rmw_xchg.wat @@ -0,0 +1,34 @@ +;;! target = "x86_64" +;;! test = "winch" + +(module + (memory 1 1 shared) + (func (export "_start") (result i32) + (i32.atomic.rmw.xchg (i32.const 0) (i32.const 42)))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x10(%r11), %r11 +;; addq $0x10, %r11 +;; cmpq %rsp, %r11 +;; ja 0x59 +;; 1c: movq %rdi, %r14 +;; subq $0x10, %rsp +;; movq %rdi, 8(%rsp) +;; movq %rsi, (%rsp) +;; movl $0x2a, %eax +;; movl $0, %ecx +;; andl $3, %ecx +;; cmpl $0, %ecx +;; jne 0x5b +;; 42: movl $0, %ecx +;; movq 0x58(%r14), %r11 +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; xchgl %eax, (%rdx) +;; addq $0x10, %rsp +;; popq %rbp +;; retq +;; 59: ud2 +;; 5b: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw16_xchgu.wat b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw16_xchgu.wat new file mode 100644 index 000000000000..af19a7fd070f --- /dev/null +++ b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw16_xchgu.wat @@ -0,0 +1,35 @@ +;;! target = "x86_64" +;;! test = "winch" + +(module + (memory 1 1 shared) + (func (export "_start") (result i64) + (i64.atomic.rmw16.xchg_u (i32.const 0) (i64.const 42)))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x10(%r11), %r11 +;; addq $0x10, %r11 +;; cmpq %rsp, %r11 +;; ja 0x62 +;; 1c: movq %rdi, %r14 +;; subq $0x10, %rsp +;; movq %rdi, 8(%rsp) +;; movq %rsi, (%rsp) +;; movq $0x2a, %rax +;; movl $0, %ecx +;; andw $1, %cx +;; cmpw $0, %cx +;; jne 0x64 +;; 46: movl $0, %ecx +;; movq 0x58(%r14), %r11 +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; xchgw %ax, (%rdx) +;; movzwq %ax, %rax +;; addq $0x10, %rsp +;; popq %rbp +;; retq +;; 62: ud2 +;; 64: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_addu.wat b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_addu.wat index cd1bf54b829d..e529fb0be1e5 100644 --- a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_addu.wat +++ b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_addu.wat @@ -4,7 +4,7 @@ (module (memory 1 1 shared) (func (export "_start") (result i64) - (i64.atomic.rmw8.add_u (i32.const 0) (i64.const 42)))) + (i64.atomic.rmw32.add_u (i32.const 0) (i64.const 42)))) ;; wasm[0]::function[0]: ;; pushq %rbp ;; movq %rsp, %rbp @@ -12,19 +12,24 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x50 +;; ja 0x5f ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movq $0x2a, %rax ;; movl $0, %ecx +;; andl $3, %ecx +;; cmpl $0, %ecx +;; jne 0x61 +;; 44: movl $0, %ecx ;; movq 0x58(%r14), %r11 ;; movq (%r11), %rdx ;; addq %rcx, %rdx -;; lock xaddb %al, (%rdx) -;; movzbq %al, %rax +;; lock xaddl %eax, (%rdx) +;; movl %eax, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 50: ud2 +;; 5f: ud2 +;; 61: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat new file mode 100644 index 000000000000..e21b5b62670c --- /dev/null +++ b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat @@ -0,0 +1,35 @@ +;;! target = "x86_64" +;;! test = "winch" + +(module + (memory 1 1 shared) + (func (export "_start") (result i64) + (i64.atomic.rmw32.xchg_u (i32.const 0) (i64.const 42)))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x10(%r11), %r11 +;; addq $0x10, %r11 +;; cmpq %rsp, %r11 +;; ja 0x5d +;; 1c: movq %rdi, %r14 +;; subq $0x10, %rsp +;; movq %rdi, 8(%rsp) +;; movq %rsi, (%rsp) +;; movq $0x2a, %rax +;; movl $0, %ecx +;; andl $3, %ecx +;; cmpl $0, %ecx +;; jne 0x5f +;; 44: movl $0, %ecx +;; movq 0x58(%r14), %r11 +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; xchgl %eax, (%rdx) +;; movl %eax, %eax +;; addq $0x10, %rsp +;; popq %rbp +;; retq +;; 5d: ud2 +;; 5f: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw8_xchgu.wat b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw8_xchgu.wat new file mode 100644 index 000000000000..ea8bc24cdd98 --- /dev/null +++ b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw8_xchgu.wat @@ -0,0 +1,30 @@ +;;! target = "x86_64" +;;! test = "winch" + +(module + (memory 1 1 shared) + (func (export "_start") (result i64) + (i64.atomic.rmw8.xchg_u (i32.const 0) (i64.const 42)))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x10(%r11), %r11 +;; addq $0x10, %r11 +;; cmpq %rsp, %r11 +;; ja 0x4e +;; 1c: movq %rdi, %r14 +;; subq $0x10, %rsp +;; movq %rdi, 8(%rsp) +;; movq %rsi, (%rsp) +;; movq $0x2a, %rax +;; movl $0, %ecx +;; movq 0x58(%r14), %r11 +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; xchgb %al, (%rdx) +;; movzbq %al, %rax +;; addq $0x10, %rsp +;; popq %rbp +;; retq +;; 4e: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw_xchg.wat b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw_xchg.wat new file mode 100644 index 000000000000..eacb439a37c5 --- /dev/null +++ b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw_xchg.wat @@ -0,0 +1,34 @@ +;;! target = "x86_64" +;;! test = "winch" + +(module + (memory 1 1 shared) + (func (export "_start") (result i64) + (i64.atomic.rmw.xchg (i32.const 0) (i64.const 42)))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x10(%r11), %r11 +;; addq $0x10, %r11 +;; cmpq %rsp, %r11 +;; ja 0x5e +;; 1c: movq %rdi, %r14 +;; subq $0x10, %rsp +;; movq %rdi, 8(%rsp) +;; movq %rsi, (%rsp) +;; movq $0x2a, %rax +;; movl $0, %ecx +;; andq $7, %rcx +;; cmpq $0, %rcx +;; jne 0x60 +;; 46: movl $0, %ecx +;; movq 0x58(%r14), %r11 +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; xchgq %rax, (%rdx) +;; addq $0x10, %rsp +;; popq %rbp +;; retq +;; 5e: ud2 +;; 60: ud2 diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs index ed32b0b471a2..cf6c0ae696d4 100644 --- a/winch/codegen/src/isa/x64/asm.rs +++ b/winch/codegen/src/isa/x64/asm.rs @@ -1147,6 +1147,31 @@ impl Assembler { }); } + pub fn xchg( + &mut self, + addr: Address, + operand: Reg, + dst: WritableReg, + size: OperandSize, + flags: MemFlags, + ) { + assert!(addr.is_offset()); + let mem = Self::to_synthetic_amode( + &addr, + &mut self.pool, + &mut self.constants, + &mut self.buffer, + flags, + ); + + self.emit(Inst::Xchg { + size: size.into(), + operand: operand.into(), + mem, + dst_old: dst.map(Into::into), + }); + } + pub fn cmp_ir(&mut self, src1: Reg, imm: i32, size: OperandSize) { let imm = RegMemImm::imm(imm as u32); diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs index 2d4fcf3b1dbe..6088891b0532 100644 --- a/winch/codegen/src/isa/x64/masm.rs +++ b/winch/codegen/src/isa/x64/masm.rs @@ -1410,6 +1410,9 @@ impl Masm for MacroAssembler { self.asm .lock_xadd(addr, operand.to_reg(), operand, size, flags); } + RmwOp::Xchg => { + self.asm.xchg(addr, operand.to_reg(), operand, size, flags); + } } if let Some(extend) = extend { diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs index 4d55b28c1d89..eee13c536828 100644 --- a/winch/codegen/src/masm.rs +++ b/winch/codegen/src/masm.rs @@ -56,6 +56,7 @@ pub(crate) enum MulWideKind { pub(crate) enum RmwOp { Add, Sub, + Xchg, } /// The direction to perform the memory move. diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs index 63e50ad0ce32..bb4054c2be6b 100644 --- a/winch/codegen/src/visitor.rs +++ b/winch/codegen/src/visitor.rs @@ -299,6 +299,13 @@ macro_rules! def_unsupported { (emit I64AtomicRmw16SubU $($rest:tt)*) => {}; (emit I64AtomicRmw32SubU $($rest:tt)*) => {}; (emit I64AtomicRmwSub $($rest:tt)*) => {}; + (emit I32AtomicRmw8XchgU $($rest:tt)*) => {}; + (emit I32AtomicRmw16XchgU $($rest:tt)*) => {}; + (emit I32AtomicRmwXchg $($rest:tt)*) => {}; + (emit I64AtomicRmw8XchgU $($rest:tt)*) => {}; + (emit I64AtomicRmw16XchgU $($rest:tt)*) => {}; + (emit I64AtomicRmw32XchgU $($rest:tt)*) => {}; + (emit I64AtomicRmwXchg $($rest:tt)*) => {}; (emit $unsupported:tt $($rest:tt)*) => {$($rest)*}; } @@ -2350,14 +2357,6 @@ where self.emit_atomic_rmw(&arg, RmwOp::Add, OperandSize::S64, None) } - fn visit_i32_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { - self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S32, None) - } - - fn visit_i64_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { - self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S64, None) - } - fn visit_i32_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { self.emit_atomic_rmw( &arg, @@ -2376,6 +2375,10 @@ where ) } + fn visit_i32_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { + self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S32, None) + } + fn visit_i64_atomic_rmw8_sub_u(&mut self, arg: MemArg) -> Self::Output { self.emit_atomic_rmw( &arg, @@ -2403,6 +2406,63 @@ where ) } + fn visit_i64_atomic_rmw_sub(&mut self, arg: MemArg) -> Self::Output { + self.emit_atomic_rmw(&arg, RmwOp::Sub, OperandSize::S64, None) + } + + fn visit_i32_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { + self.emit_atomic_rmw( + &arg, + RmwOp::Xchg, + OperandSize::S8, + Some(ExtendKind::I32Extend8U), + ) + } + + fn visit_i32_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { + self.emit_atomic_rmw( + &arg, + RmwOp::Xchg, + OperandSize::S16, + Some(ExtendKind::I32Extend16U), + ) + } + + fn visit_i32_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { + self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S32, None) + } + + fn visit_i64_atomic_rmw8_xchg_u(&mut self, arg: MemArg) -> Self::Output { + self.emit_atomic_rmw( + &arg, + RmwOp::Xchg, + OperandSize::S8, + Some(ExtendKind::I64Extend8U), + ) + } + + fn visit_i64_atomic_rmw16_xchg_u(&mut self, arg: MemArg) -> Self::Output { + self.emit_atomic_rmw( + &arg, + RmwOp::Xchg, + OperandSize::S16, + Some(ExtendKind::I64Extend16U), + ) + } + + fn visit_i64_atomic_rmw32_xchg_u(&mut self, arg: MemArg) -> Self::Output { + self.emit_atomic_rmw( + &arg, + RmwOp::Xchg, + OperandSize::S32, + Some(ExtendKind::I64Extend32U), + ) + } + + fn visit_i64_atomic_rmw_xchg(&mut self, arg: MemArg) -> Self::Output { + self.emit_atomic_rmw(&arg, RmwOp::Xchg, OperandSize::S64, None) + } + wasmparser::for_each_visit_operator!(def_unsupported); } From edd36497eaf0e261d36a84fc7fdff4bdf70b60cb Mon Sep 17 00:00:00 2001 From: adhoc Date: Fri, 17 Jan 2025 01:53:05 +0100 Subject: [PATCH 2/2] fix test post rebase --- tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_addu.wat | 7 +++---- .../disas/winch/x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_addu.wat b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_addu.wat index e529fb0be1e5..40d2bb6f5171 100644 --- a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_addu.wat +++ b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_addu.wat @@ -12,7 +12,7 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x5f +;; ja 0x5d ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) @@ -21,15 +21,14 @@ ;; movl $0, %ecx ;; andl $3, %ecx ;; cmpl $0, %ecx -;; jne 0x61 +;; jne 0x5f ;; 44: movl $0, %ecx ;; movq 0x58(%r14), %r11 ;; movq (%r11), %rdx ;; addq %rcx, %rdx ;; lock xaddl %eax, (%rdx) -;; movl %eax, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq +;; 5d: ud2 ;; 5f: ud2 -;; 61: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat index e21b5b62670c..a027d17380c4 100644 --- a/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat +++ b/tests/disas/winch/x64/atomic/rmw/i64_atomic_rmw32_xchgu.wat @@ -12,7 +12,7 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x5d +;; ja 0x5b ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) @@ -21,15 +21,14 @@ ;; movl $0, %ecx ;; andl $3, %ecx ;; cmpl $0, %ecx -;; jne 0x5f +;; jne 0x5d ;; 44: movl $0, %ecx ;; movq 0x58(%r14), %r11 ;; movq (%r11), %rdx ;; addq %rcx, %rdx ;; xchgl %eax, (%rdx) -;; movl %eax, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq +;; 5b: ud2 ;; 5d: ud2 -;; 5f: ud2