s390x: Implement tls_value (bytecodealliance#4616)

Implement the tls_value for s390 in the ELF general-dynamic mode.

Notable differences to the x86_64 implementation are:
- We use a __tls_get_offset libcall instead of __tls_get_addr.
- The current thread pointer (stored in a pair of access registers)
  needs to be added to the result of __tls_get_offset.
- __tls_get_offset has a variant ABI that requires the address of
  the GOT (global offset table) is passed in %r12.

This means we need a new libcall entries for __tls_get_offset.
In addition, we also need a way to access _GLOBAL_OFFSET_TABLE_.
The latter is a "magic" symbol with a well-known name defined
by the ABI and recognized by the linker.  This patch introduces
a new ExternalName::KnownSymbol variant to support such names
(originally due to @afonso360).

We also need to emit a relocation on a symbol placed in a
constant pool, as well as an extra relocation on the call
to __tls_get_offset required for TLS linker optimization.

Needed by the cg_clif frontend.

cranelift/codegen/src/binemit/mod.rs

@@ -59,6 +59,11 @@ pub enum Reloc {
     /// Set the add immediate field to the low 12 bits of the final address. Does not check for overflow.
     /// This is equivalent to `R_AARCH64_TLSGD_ADD_LO12_NC` in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#relocations-for-thread-local-storage)
     Aarch64TlsGdAddLo12Nc,
+
+    /// s390x TLS GD64 - 64-bit offset of tls_index for GD symbol in GOT
+    S390xTlsGd64,
+    /// s390x TLS GDCall - marker to enable optimization of TLS calls
+    S390xTlsGdCall,
 }
 
 impl fmt::Display for Reloc {
@@ -79,6 +84,8 @@ impl fmt::Display for Reloc {
             Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"),
             Self::Aarch64TlsGdAdrPage21 => write!(f, "Aarch64TlsGdAdrPage21"),
             Self::Aarch64TlsGdAddLo12Nc => write!(f, "Aarch64TlsGdAddLo12Nc"),
+            Self::S390xTlsGd64 => write!(f, "TlsGd64"),
+            Self::S390xTlsGdCall => write!(f, "TlsGdCall"),
         }
     }
 }

cranelift/codegen/src/ir/extname.rs

@@ -4,7 +4,7 @@
 //! function. The name of an external declaration doesn't have any meaning to
 //! Cranelift, which compiles functions independently.
 
-use crate::ir::LibCall;
+use crate::ir::{KnownSymbol, LibCall};
 use core::cmp;
 use core::fmt::{self, Write};
 use core::str::FromStr;
@@ -46,6 +46,8 @@ pub enum ExternalName {
     },
     /// A well-known runtime library function.
     LibCall(LibCall),
+    /// A well-known symbol.
+    KnownSymbol(KnownSymbol),
 }
 
 impl ExternalName {
@@ -104,6 +106,7 @@ impl fmt::Display for ExternalName {
                 Ok(())
             }
             Self::LibCall(lc) => write!(f, "%{}", lc),
+            Self::KnownSymbol(ks) => write!(f, "%{}", ks),
         }
     }
 }
@@ -112,11 +115,18 @@ impl FromStr for ExternalName {
     type Err = ();
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        // Try to parse as a libcall name, otherwise it's a test case.
-        match s.parse() {
-            Ok(lc) => Ok(Self::LibCall(lc)),
-            Err(_) => Ok(Self::testcase(s.as_bytes())),
+        // Try to parse as a known symbol
+        if let Ok(ks) = s.parse() {
+            return Ok(Self::KnownSymbol(ks));
         }
+
+        // Try to parse as a libcall name
+        if let Ok(lc) = s.parse() {
+            return Ok(Self::LibCall(lc));
+        }
+
+        // Otherwise its a test case name
+        Ok(Self::testcase(s.as_bytes()))
     }
 }
 

cranelift/codegen/src/ir/known_symbol.rs

@@ -0,0 +1,42 @@
+use core::fmt;
+use core::str::FromStr;
+#[cfg(feature = "enable-serde")]
+use serde::{Deserialize, Serialize};
+
+/// A well-known symbol.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+pub enum KnownSymbol {
+    /// ELF well-known linker symbol _GLOBAL_OFFSET_TABLE_
+    ElfGlobalOffsetTable,
+}
+
+impl fmt::Display for KnownSymbol {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        fmt::Debug::fmt(self, f)
+    }
+}
+
+impl FromStr for KnownSymbol {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "ElfGlobalOffsetTable" => Ok(Self::ElfGlobalOffsetTable),
+            _ => Err(()),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parsing() {
+        assert_eq!(
+            "ElfGlobalOffsetTable".parse(),
+            Ok(KnownSymbol::ElfGlobalOffsetTable)
+        );
+    }
+}

cranelift/codegen/src/ir/libcall.rs

@@ -66,6 +66,8 @@ pub enum LibCall {
 
     /// Elf __tls_get_addr
     ElfTlsGetAddr,
+    /// Elf __tls_get_offset
+    ElfTlsGetOffset,
     // When adding a new variant make sure to add it to `all_libcalls` too.
 }
 
@@ -104,6 +106,7 @@ impl FromStr for LibCall {
             "Memcmp" => Ok(Self::Memcmp),
 
             "ElfTlsGetAddr" => Ok(Self::ElfTlsGetAddr),
+            "ElfTlsGetOffset" => Ok(Self::ElfTlsGetOffset),
             _ => Err(()),
         }
     }
@@ -173,6 +176,7 @@ impl LibCall {
             Memmove,
             Memcmp,
             ElfTlsGetAddr,
+            ElfTlsGetOffset,
         ]
     }
 
@@ -214,7 +218,8 @@ impl LibCall {
             | LibCall::Memset
             | LibCall::Memmove
             | LibCall::Memcmp
-            | LibCall::ElfTlsGetAddr => unimplemented!(),
+            | LibCall::ElfTlsGetAddr
+            | LibCall::ElfTlsGetOffset => unimplemented!(),
         }
 
         sig

cranelift/codegen/src/ir/mod.rs

@@ -15,6 +15,7 @@ mod heap;
 pub mod immediates;
 pub mod instructions;
 pub mod jumptable;
+pub(crate) mod known_symbol;
 pub mod layout;
 pub(crate) mod libcall;
 mod memflags;
@@ -50,6 +51,7 @@ pub use crate::ir::instructions::{
     InstructionData, Opcode, ValueList, ValueListPool, VariableArgs,
 };
 pub use crate::ir::jumptable::JumpTableData;
+pub use crate::ir::known_symbol::KnownSymbol;
 pub use crate::ir::layout::Layout;
 pub use crate::ir::libcall::{get_probestack_funcref, LibCall};
 pub use crate::ir::memflags::{Endianness, MemFlags};

cranelift/codegen/src/isa/s390x/inst.isle

@@ -424,6 +424,17 @@
       (rd WritableReg)
       (imm UImm32Shifted))
 
+    ;; Load 32-bit access register into GPR.
+    (LoadAR
+      (rd WritableReg)
+      (ar u8))
+
+    ;; Insert 32-bit access register into low half of a GPR.
+    ;; (Identical operation to LoadAR, but considers rd to be use/def.)
+    (InsertAR
+      (rd WritableReg)
+      (ar u8))
+
     ;; A sign- or zero-extend operation.
     (Extend
       (rd WritableReg)
@@ -857,11 +868,10 @@
       (ridx Reg)
       (targets VecMachLabel))
 
-    ;; Load an inline symbol reference with RelocDistance::Far.
-    (LoadExtNameFar
+    ;; Load an inline symbol reference with relocation.
+    (LoadSymbolReloc
       (rd WritableReg)
-      (name BoxExternalName)
-      (offset i64))
+      (symbol_reloc BoxSymbolReloc))
 
     ;; Load address referenced by `mem` into `rd`.
     (LoadAddr
@@ -903,6 +913,23 @@
 (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo))
 (type VecMachLabel extern (enum))
 
+;; A symbol reference carrying relocation information.
+(type SymbolReloc
+  (enum
+    ;; Absolute symbol reference (with optional offset).
+    (Absolute
+      (name ExternalName)
+      (offset i64))
+    ;; Reference to a TLS symbol in general-dynamic mode.
+    (TlsGd
+      (name ExternalName))))
+
+;; Boxed version of SymbolReloc to save space.
+(type BoxSymbolReloc (primitive BoxSymbolReloc))
+(decl box_symbol_reloc (SymbolReloc) BoxSymbolReloc)
+(extern constructor box_symbol_reloc box_symbol_reloc)
+(convert SymbolReloc BoxSymbolReloc box_symbol_reloc)
+
 ;; An ALU operation.
 (type ALUOp
   (enum
@@ -1613,6 +1640,9 @@
 (decl memarg_symbol (ExternalName i32 MemFlags) MemArg)
 (extern constructor memarg_symbol memarg_symbol)
 
+(decl memarg_got () MemArg)
+(extern constructor memarg_got memarg_got)
+
 ;; Create a MemArg refering to a stack address formed by
 ;; adding a base (relative to SP) and an offset.
 (decl memarg_stack_off (i64 i64) MemArg)
@@ -2120,6 +2150,20 @@
 (rule (mvc dst src len_minus_one)
       (SideEffectNoResult.Inst (MInst.Mvc dst src len_minus_one)))
 
+;; Helper for emitting `MInst.LoadAR` instructions.
+(decl load_ar (u8) Reg)
+(rule (load_ar ar)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.LoadAR dst ar))))
+        dst))
+
+;; Helper for emitting `MInst.InsertAR` instructions.
+(decl insert_ar (Reg u8) Reg)
+(rule (insert_ar src ar)
+      (let ((dst WritableReg (copy_writable_reg $I64 src))
+            (_ Unit (emit (MInst.InsertAR dst ar))))
+        dst))
+
 ;; Helper for emitting `MInst.FpuRR` instructions.
 (decl fpu_rr (Type FPUOp1 Reg) Reg)
 (rule (fpu_rr ty op src)
@@ -2393,12 +2437,11 @@
             (_ Unit (emit (MInst.VecReplicateLane size dst src lane_imm))))
         dst))
 
-;; Helper for emitting `MInst.LoadExtNameFar` instructions.
-(decl load_ext_name_far (ExternalName i64) Reg)
-(rule (load_ext_name_far name offset)
+;; Helper for emitting `MInst.LoadSymbolReloc` instructions.
+(decl load_symbol_reloc (SymbolReloc) Reg)
+(rule (load_symbol_reloc symbol_reloc)
       (let ((dst WritableReg (temp_writable_reg $I64))
-            (boxed_name BoxExternalName (box_external_name name))
-            (_ Unit (emit (MInst.LoadExtNameFar dst boxed_name offset))))
+            (_ Unit (emit (MInst.LoadSymbolReloc dst symbol_reloc))))
         dst))
 
 ;; Helper for emitting `MInst.LoadAddr` instructions.
@@ -3405,6 +3448,9 @@
 (decl lib_call_info_memcpy () LibCallInfo)
 (extern constructor lib_call_info_memcpy lib_call_info_memcpy)
 
+(decl lib_call_info_tls_get_offset (SymbolReloc) LibCallInfo)
+(extern constructor lib_call_info_tls_get_offset lib_call_info_tls_get_offset)
+
 (decl lib_call_info (LibCallInfo) BoxCallInfo)
 (extern constructor lib_call_info lib_call_info)
 

cranelift/codegen/src/isa/s390x/inst/emit.rs

@@ -2258,17 +2258,25 @@ impl MachInstEmit for Inst {
                 };
                 put(sink, &enc_ril_a(opcode, rd.to_reg(), imm.bits));
             }
-            &Inst::LoadExtNameFar {
+            &Inst::LoadAR { rd, ar } | &Inst::InsertAR { rd, ar } => {
+                let rd = allocs.next_writable(rd);
+                let opcode = 0xb24f; // EAR
+                put(sink, &enc_rre(opcode, rd.to_reg(), gpr(ar)));
+            }
+            &Inst::LoadSymbolReloc {
                 rd,
-                ref name,
-                offset,
+                ref symbol_reloc,
             } => {
                 let rd = allocs.next_writable(rd);
 
                 let opcode = 0xa75; // BRAS
                 let reg = writable_spilltmp_reg().to_reg();
                 put(sink, &enc_ri_b(opcode, reg, 12));
-                sink.add_reloc(Reloc::Abs8, name, offset);
+                let (reloc, name, offset) = match &**symbol_reloc {
+                    SymbolReloc::Absolute { name, offset } => (Reloc::Abs8, name, *offset),
+                    SymbolReloc::TlsGd { name } => (Reloc::S390xTlsGd64, name, 0),
+                };
+                sink.add_reloc(reloc, name, offset);
                 sink.put8(0);
                 let inst = Inst::Load64 {
                     rd,
@@ -3198,6 +3206,15 @@ impl MachInstEmit for Inst {
             &Inst::Call { link, ref info } => {
                 let link = allocs.next_writable(link);
 
+                // Add relocation for TLS libcalls to enable linker optimizations.
+                match &info.tls_symbol {
+                    None => {}
+                    Some(SymbolReloc::TlsGd { name }) => {
+                        sink.add_reloc(Reloc::S390xTlsGdCall, name, 0)
+                    }
+                    _ => unreachable!(),
+                }
+
                 let opcode = 0xc05; // BRASL
                 let reloc = Reloc::S390xPCRel32Dbl;
                 if let Some(s) = state.take_stack_map() {

cranelift/codegen/src/isa/s390x/inst/emit_tests.rs

@@ -6828,6 +6828,7 @@ fn test_s390x_binemit() {
                 opcode: Opcode::Call,
                 caller_callconv: CallConv::SystemV,
                 callee_callconv: CallConv::SystemV,
+                tls_symbol: None,
             }),
         },
         "C0E500000000",