Implement atomic_ref (#2, p0019r8) (#843)

Co-authored-by: Billy O'Neal <[email protected]>
Co-authored-by: Adam Bucior <[email protected]>
Co-authored-by: Stephan T. Lavavej <[email protected]>

stl/inc/xatomic.h

@@ -11,6 +11,9 @@
 
 #include <intrin0.h>
 #include <type_traits>
+#if defined(_WIN64) && (_MSC_FULL_VER < 192829203) // TRANSITION
+#include <intrin.h> // Visual Studio 2019 to define 128-bit CAS in <intrin0.h>
+#endif // defined(_WIN64) && (_MSC_FULL_VER < 192829203), TRANSITION
 
 #pragma pack(push, _CRT_PACKING)
 #pragma warning(push, _STL_WARNING_LEVEL)

stl/inc/yvals.h

@@ -208,6 +208,21 @@ _STL_DISABLE_CLANG_WARNINGS
 #define _STL_INTERNAL_STATIC_ASSERT(...)
 #endif // _ENABLE_STL_INTERNAL_CHECK
 
+#ifndef _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK
+#ifdef _DEBUG
+#define _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK 1
+#else // ^^^ _DEBUG ^^^ // vvv !_DEBUG vvv
+#define _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK 0
+#endif // _DEBUG
+#endif // _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK
+
+#if _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK
+#define _ATOMIC_REF_CHECK_ALIGNMENT(cond, mesg) _STL_VERIFY(cond, mesg)
+#else
+#define _ATOMIC_REF_CHECK_ALIGNMENT(cond, mesg) _Analysis_assume_(cond)
+#endif
+
+
 #include <use_ansi.h>
 
 #define _WARNING_MESSAGE(NUMBER, MESSAGE) __FILE__ "(" _CRT_STRINGIZE(__LINE__) "): warning " NUMBER ": " MESSAGE
@@ -306,6 +321,22 @@ _STL_DISABLE_CLANG_WARNINGS
 #define _LOCK_DEBUG          3
 #define _LOCK_AT_THREAD_EXIT 4
 
+#ifndef _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B
+#if _STL_WIN32_WINNT >= _STL_WIN32_WINNT_WINBLUE && defined(_WIN64)
+#define _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B 1
+#else // ^^^ modern 64-bit // less modern or 32-bit vvv
+#define _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B 0
+#endif // _STL_WIN32_WINNT >= _STL_WIN32_WINNT_WINBLUE && defined(_WIN64)
+#endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B
+
+#if _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 0 && defined(_M_ARM64)
+#error ARM64 requires _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B to be 1.
+#endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 0 && defined(_M_ARM64)
+
+#if _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 1 && !defined(_WIN64)
+#error _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 1 requires 64-bit.
+#endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 1 && !defined(_WIN64)
+
 #ifdef __cplusplus
 _STD_BEGIN
 enum _Uninitialized { // tag for suppressing initialization

stl/inc/yvals_core.h

@@ -131,6 +131,7 @@
 // Other C++17 deprecation warnings
 
 // _HAS_CXX20 directly controls:
+// P0019R8 atomic_ref
 // P0020R6 atomic<float>, atomic<double>, atomic<long double>
 // P0122R7 <span>
 // P0202R3 constexpr For <algorithm> And exchange()
@@ -147,6 +148,7 @@
 // P0482R6 Library Support For char8_t
 //     (mbrtoc8 and c8rtomb not yet implemented)
 // P0487R1 Fixing operator>>(basic_istream&, CharT*)
+// P0528R3 Atomic Compare-And-Exchange With Padding Bits
 // P0550R2 remove_cvref
 // P0553R4 <bit> Rotating And Counting Functions
 // P0556R3 <bit> Integral Power-Of-2 Operations (renamed by P1956R1)
@@ -182,6 +184,9 @@
 //     (except the std::invoke function which is implemented in C++17)
 // P1085R2 Removing span Comparisons
 // P1115R3 erase()/erase_if() Return size_type
+// P1123R0 Atomic Compare-And-Exchange With Padding Bits For atomic_ref
+// P1135R6 The C++20 Synchronization Library
+//     (partially implemented)
 // P1207R4 Movability Of Single-Pass Iterators
 //     (partially implemented)
 // P1209R0 erase_if(), erase()
@@ -207,6 +212,7 @@
 // P1907R2 ranges::ssize
 // P1956R1 <bit> has_single_bit(), bit_ceil(), bit_floor(), bit_width()
 // P1959R0 Removing weak_equality And strong_equality
+// P1960R0 atomic_ref Cleanup
 // P1964R2 Replacing boolean With boolean-testable
 // P1976R2 Explicit Constructors For Fixed-Extent span From Dynamic-Extent Ranges
 // P2091R0 Fixing Issues With Range Access CPOs
@@ -1137,6 +1143,7 @@
 #define __cpp_lib_atomic_flag_test              201907L
 #define __cpp_lib_atomic_float                  201711L
 #define __cpp_lib_atomic_lock_free_type_aliases 201907L
+#define __cpp_lib_atomic_ref                    201806L
 #define __cpp_lib_atomic_shared_ptr             201711L
 #define __cpp_lib_atomic_wait                   201907L
 #define __cpp_lib_bind_front                    201907L
@@ -1262,13 +1269,18 @@ compiler option, or define _ALLOW_RTCc_IN_STL to acknowledge that you have recei
 #error In yvals_core.h, defined(MRTDLL) implies defined(_M_CEE_PURE); !defined(_M_CEE_PURE) implies !defined(MRTDLL)
 #endif // defined(MRTDLL) && !defined(_M_CEE_PURE)
 
-#define _STL_WIN32_WINNT_WINXP 0x0501 // _WIN32_WINNT_WINXP from sdkddkver.h
-#define _STL_WIN32_WINNT_VISTA 0x0600 // _WIN32_WINNT_VISTA from sdkddkver.h
-#define _STL_WIN32_WINNT_WIN8  0x0602 // _WIN32_WINNT_WIN8 from sdkddkver.h
+#define _STL_WIN32_WINNT_WINXP   0x0501 // _WIN32_WINNT_WINXP from sdkddkver.h
+#define _STL_WIN32_WINNT_VISTA   0x0600 // _WIN32_WINNT_VISTA from sdkddkver.h
+#define _STL_WIN32_WINNT_WIN8    0x0602 // _WIN32_WINNT_WIN8 from sdkddkver.h
+#define _STL_WIN32_WINNT_WINBLUE 0x0603 // _WIN32_WINNT_WINBLUE from sdkddkver.h
+#define _STL_WIN32_WINNT_WIN10   0x0A00 // _WIN32_WINNT_WIN10 from sdkddkver.h
 
 // Note that the STL DLL builds will set this to XP for ABI compatibility with VS2015 which supported XP.
 #ifndef _STL_WIN32_WINNT
-#if defined(_M_ARM) || defined(_M_ARM64) || defined(_ONECORE) || defined(_CRT_APP)
+#if defined(_M_ARM64)
+// The first ARM64 Windows was Windows 10
+#define _STL_WIN32_WINNT _STL_WIN32_WINNT_WIN10
+#elif defined(_M_ARM) || defined(_ONECORE) || defined(_CRT_APP)
 // The first ARM or OneCore or App Windows was Windows 8
 #define _STL_WIN32_WINNT _STL_WIN32_WINNT_WIN8
 #else // ^^^ default to Win8 // default to Vista vvv

stl/src/atomic_wait.cpp

@@ -198,6 +198,24 @@ namespace {
         }
     }
 #endif // _ATOMIC_WAIT_ON_ADDRESS_STATICALLY_AVAILABLE
+
+    _NODISCARD unsigned char __std_atomic_compare_exchange_128_fallback(_Inout_bytecount_(16) long long* _Destination,
+        _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow,
+        _Inout_bytecount_(16) long long* _ComparandResult) noexcept {
+        static SRWLOCK _Mtx = SRWLOCK_INIT;
+        _SrwLock_guard _Guard{_Mtx};
+        if (_Destination[0] == _ComparandResult[0] && _Destination[1] == _ComparandResult[1]) {
+            _ComparandResult[0] = _Destination[0];
+            _ComparandResult[1] = _Destination[1];
+            _Destination[0]     = _ExchangeLow;
+            _Destination[1]     = _ExchangeHigh;
+            return static_cast<unsigned char>(true);
+        } else {
+            _ComparandResult[0] = _Destination[0];
+            _ComparandResult[1] = _Destination[1];
+            return static_cast<unsigned char>(false);
+        }
+    }
 } // unnamed namespace
 
 
@@ -339,4 +357,62 @@ __std_atomic_api_level __stdcall __std_atomic_set_api_level(__std_atomic_api_lev
     return _Acquire_wait_functions();
 #endif // !_ATOMIC_WAIT_ON_ADDRESS_STATICALLY_AVAILABLE
 }
+
+#pragma warning(push)
+#pragma warning(disable : 4324) // structure was padded due to alignment specifier
+_Smtx_t* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept {
+    constexpr size_t _Table_size_power = 8;
+    constexpr size_t _Table_size       = 1 << _Table_size_power;
+    constexpr size_t _Table_index_mask = _Table_size - 1;
+
+    struct alignas(std::hardware_destructive_interference_size) _Table_entry {
+        _Smtx_t _Mutex;
+    };
+
+    static _Table_entry _Table[_Table_size]{};
+
+    auto _Index = reinterpret_cast<std::uintptr_t>(_Key);
+    _Index ^= _Index >> (_Table_size_power * 2);
+    _Index ^= _Index >> _Table_size_power;
+    return &_Table[_Index & _Table_index_mask]._Mutex;
+}
+#pragma warning(pop)
+
+_NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128(_Inout_bytecount_(16) long long* _Destination,
+    _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow,
+    _Inout_bytecount_(16) long long* _ComparandResult) noexcept {
+#if !defined(_WIN64)
+    return __std_atomic_compare_exchange_128_fallback(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult);
+#elif _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 1
+    return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult);
+#else // ^^^ _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 1 // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 0 vvv
+    if (__std_atomic_has_cmpxchg16b()) {
+        return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult);
+    }
+
+    return __std_atomic_compare_exchange_128_fallback(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult);
+#endif // ^^^ _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 0
+}
+
+_NODISCARD char __stdcall __std_atomic_has_cmpxchg16b() noexcept {
+#if !defined(_WIN64)
+    return false;
+#elif _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 1
+    return true;
+#else // ^^^ _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 1 // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 0 vvv
+    constexpr char _Cmpxchg_Absent  = 0;
+    constexpr char _Cmpxchg_Present = 1;
+    constexpr char _Cmpxchg_Unknown = 2;
+
+    static std::atomic<char> _Cached_value{_Cmpxchg_Unknown};
+
+    char _Value = _Cached_value.load(std::memory_order_relaxed);
+    if (_Value == _Cmpxchg_Unknown) {
+        _Value = IsProcessorFeaturePresent(PF_COMPARE_EXCHANGE128) ? _Cmpxchg_Present : _Cmpxchg_Absent;
+        _Cached_value.store(_Value, std::memory_order_relaxed);
+    }
+
+    return _Value;
+#endif // ^^^ _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B == 0
+}
 _END_EXTERN_C

stl/src/msvcp_atomic_wait.src

@@ -6,14 +6,17 @@
 LIBRARY LIBRARYNAME
 
 EXPORTS
-    __std_atomic_wait_get_deadline
-    __std_atomic_wait_get_remaining_timeout
+    __std_atomic_compare_exchange_128
+    __std_atomic_get_mutex
+    __std_atomic_has_cmpxchg16b
     __std_atomic_notify_all_direct
     __std_atomic_notify_all_indirect
     __std_atomic_notify_one_direct
     __std_atomic_notify_one_indirect
     __std_atomic_set_api_level
     __std_atomic_wait_direct
+    __std_atomic_wait_get_deadline
+    __std_atomic_wait_get_remaining_timeout
     __std_atomic_wait_indirect
     __std_bulk_submit_threadpool_work
     __std_close_threadpool_work

tests/libcxx/expected_results.txt

@@ -49,6 +49,7 @@ std/containers/unord/unord.map/unord.map.modifiers/insert_and_emplace_allocator_
 std/containers/unord/unord.set/insert_and_emplace_allocator_requirements.pass.cpp FAIL
 
 # libc++ doesn't yet implement P1423R3, so it expects an old value for `__cpp_lib_char8_t`
+std/language.support/support.limits/support.limits.general/atomic.version.pass.cpp FAIL
 std/language.support/support.limits/support.limits.general/filesystem.version.pass.cpp FAIL
 std/language.support/support.limits/support.limits.general/istream.version.pass.cpp FAIL
 std/language.support/support.limits/support.limits.general/limits.version.pass.cpp FAIL
@@ -252,9 +253,6 @@ std/utilities/memory/default.allocator/allocator.members/allocate.verify.cpp SKI
 
 
 # *** MISSING STL FEATURES ***
-# C++20 P0019R8 "atomic_ref"
-std/language.support/support.limits/support.limits.general/atomic.version.pass.cpp FAIL
-
 # C++20 P0355R7 "<chrono> Calendars And Time Zones"
 std/utilities/time/days.pass.cpp FAIL
 std/utilities/time/months.pass.cpp FAIL

tests/libcxx/skipped_tests.txt

@@ -49,6 +49,7 @@ containers\unord\unord.map\unord.map.modifiers\insert_and_emplace_allocator_requ
 containers\unord\unord.set\insert_and_emplace_allocator_requirements.pass.cpp
 
 # libc++ doesn't yet implement P1423R3, so it expects an old value for `__cpp_lib_char8_t`
+language.support\support.limits\support.limits.general\atomic.version.pass.cpp
 language.support\support.limits\support.limits.general\filesystem.version.pass.cpp
 language.support\support.limits\support.limits.general\istream.version.pass.cpp
 language.support\support.limits\support.limits.general\limits.version.pass.cpp
@@ -252,9 +253,6 @@ utilities\memory\default.allocator\allocator.members\allocate.verify.cpp
 
 
 # *** MISSING STL FEATURES ***
-# C++20 P0019R8 "atomic_ref"
-language.support\support.limits\support.limits.general\atomic.version.pass.cpp
-
 # C++20 P0355R7 "<chrono> Calendars And Time Zones"
 utilities\time\days.pass.cpp
 utilities\time\months.pass.cpp

tests/std/include/test_atomic_wait.hpp

@@ -10,8 +10,8 @@
 #include <string.h>
 #include <thread>
 
-template <class UnderlyingType>
-void test_atomic_wait_func(const UnderlyingType old_value, const UnderlyingType new_value,
+template <template <class> class Template, class UnderlyingType>
+void test_atomic_wait_func_impl(UnderlyingType& old_value, const UnderlyingType new_value,
     const std::chrono::steady_clock::duration waiting_duration) {
     constexpr int seq_max_size = 10;
     char seq[seq_max_size + 1];
@@ -22,7 +22,7 @@ void test_atomic_wait_func(const UnderlyingType old_value, const UnderlyingType
         *p = ch;
     };
 
-    std::atomic<UnderlyingType> a{old_value};
+    Template<UnderlyingType> a(old_value);
     a.wait(new_value);
 
     add_seq('1');
@@ -64,9 +64,24 @@ void test_atomic_wait_func(const UnderlyingType old_value, const UnderlyingType
 }
 
 template <class UnderlyingType>
-void test_notify_all_notifies_all(const UnderlyingType old_value, const UnderlyingType new_value,
+void test_atomic_wait_func(UnderlyingType old_value, const UnderlyingType new_value,
+    const std::chrono::steady_clock::duration waiting_duration) {
+    test_atomic_wait_func_impl<std::atomic, UnderlyingType>(old_value, new_value, waiting_duration);
+    alignas(std::atomic_ref<UnderlyingType>::required_alignment) UnderlyingType old_value_for_ref = old_value;
+    test_atomic_wait_func_impl<std::atomic_ref, UnderlyingType>(old_value_for_ref, new_value, waiting_duration);
+}
+
+template <class UnderlyingType>
+void test_atomic_wait_func_ptr(UnderlyingType old_value, const UnderlyingType new_value,
     const std::chrono::steady_clock::duration waiting_duration) {
-    std::atomic<UnderlyingType> c{old_value};
+    test_atomic_wait_func_impl<std::atomic, UnderlyingType>(old_value, new_value, waiting_duration);
+}
+
+
+template <template <class> class Template, class UnderlyingType>
+void test_notify_all_notifies_all_impl(UnderlyingType& old_value, const UnderlyingType new_value,
+    const std::chrono::steady_clock::duration waiting_duration) {
+    Template<UnderlyingType> c(old_value);
     const auto waitFn = [&c, old_value] { c.wait(old_value); };
 
     std::thread w1{waitFn};
@@ -83,16 +98,31 @@ void test_notify_all_notifies_all(const UnderlyingType old_value, const Underlyi
 }
 
 template <class UnderlyingType>
-void test_pad_bits(const std::chrono::steady_clock::duration waiting_duration) {
-    UnderlyingType old_value;
+void test_notify_all_notifies_all(UnderlyingType old_value, const UnderlyingType new_value,
+    const std::chrono::steady_clock::duration waiting_duration) {
+    test_notify_all_notifies_all_impl<std::atomic, UnderlyingType>(old_value, new_value, waiting_duration);
+    alignas(std::atomic_ref<UnderlyingType>::required_alignment) UnderlyingType old_value_for_ref = old_value;
+    test_notify_all_notifies_all_impl<std::atomic_ref, UnderlyingType>(old_value_for_ref, new_value, waiting_duration);
+}
+
+template <class UnderlyingType>
+void test_notify_all_notifies_all_ptr(UnderlyingType old_value, const UnderlyingType new_value,
+    const std::chrono::steady_clock::duration waiting_duration) {
+    test_notify_all_notifies_all_impl<std::atomic, UnderlyingType>(old_value, new_value, waiting_duration);
+}
+
+
+template <template <class> class Template, class UnderlyingType>
+void test_pad_bits_impl(const std::chrono::steady_clock::duration waiting_duration) {
+    alignas(std::atomic_ref<UnderlyingType>::required_alignment) UnderlyingType old_value;
     memset(&old_value, 0x66, sizeof(UnderlyingType));
     old_value.set(1);
 
     UnderlyingType same_old_value;
     memset(&same_old_value, 0x99, sizeof(UnderlyingType));
     same_old_value.set(1);
 
-    std::atomic<UnderlyingType> c(old_value);
+    Template<UnderlyingType> c(old_value);
 
     bool trigger      = false;
     const auto waitFn = [&c, same_old_value, &trigger] {
@@ -123,6 +153,12 @@ void test_pad_bits(const std::chrono::steady_clock::duration waiting_duration) {
     w1.join();
 }
 
+template <class UnderlyingType>
+void test_pad_bits(const std::chrono::steady_clock::duration waiting_duration) {
+    test_pad_bits_impl<std::atomic, UnderlyingType>(waiting_duration);
+    test_pad_bits_impl<std::atomic_ref, UnderlyingType>(waiting_duration);
+}
+
 struct two_shorts {
     short a;
     short b;
@@ -182,9 +218,9 @@ inline void test_atomic_wait() {
     test_atomic_wait_func(three_chars{1, 1, 3}, three_chars{1, 2, 3}, waiting_duration);
     test_atomic_wait_func(big_char_like{'a'}, big_char_like{'b'}, waiting_duration);
 
-    test_atomic_wait_func(std::make_shared<int>('a'), std::make_shared<int>('b'), waiting_duration);
-    test_atomic_wait_func(
-        std::weak_ptr{std::make_shared<int>('a')}, std::weak_ptr{std::make_shared<int>('b')}, waiting_duration);
+    test_atomic_wait_func_ptr(std::make_shared<int>('a'), std::make_shared<int>('a'), waiting_duration);
+    test_atomic_wait_func_ptr(
+        std::weak_ptr{std::make_shared<int>('a')}, std::weak_ptr{std::make_shared<int>('a')}, waiting_duration);
 
     test_notify_all_notifies_all<char>(1, 2, waiting_duration);
     test_notify_all_notifies_all<signed char>(1, 2, waiting_duration);

tests/std/test.lst

@@ -167,6 +167,7 @@ tests\GH_001017_discrete_distribution_out_of_range
 tests\GH_001086_partial_sort_copy
 tests\LWG2597_complex_branch_cut
 tests\LWG3018_shared_ptr_function
+tests\P0019R8_atomic_ref
 tests\P0024R2_parallel_algorithms_adjacent_difference
 tests\P0024R2_parallel_algorithms_adjacent_find
 tests\P0024R2_parallel_algorithms_all_of

tests/std/tests/P0019R8_atomic_ref/env.lst

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+RUNALL_INCLUDE ..\usual_latest_matrix.lst