Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Vectorize reverse_copy() #804

Merged
merged 15 commits into from
Jul 27, 2020
Merged

Vectorize reverse_copy() #804

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
aa76476
Add vectorized reverse copy functions
pawREP May 6, 2020
2d73df7
Vectorize reverse_copy
pawREP May 6, 2020
7d80f16
Replace overloads of _Advance_bytes and _Byte_length with templates
pawREP May 6, 2020
4add28b
Revision 1
pawREP May 7, 2020
2e19adb
Remove _USE_STD_VECTOR_ALGORITHMS define
pawREP May 7, 2020
a780f10
Rename __std_reverse_trivially_copyable_X
pawREP May 7, 2020
b00bc06
Merge remote-tracking branch 'origin/master' into Vectorize-reverse_copy
BillyONeal Jul 21, 2020
e63348e
Add test cases.
BillyONeal Jul 22, 2020
d516b5e
Return _Advance_bytes to being an overload set rather than a template.
BillyONeal Jul 22, 2020
54f9b22
Simplify by vectorizing _HAS_IF_CONSTEXPR only.
StephanTLavavej Jul 22, 2020
49a0f5f
Fix bug: As the comment says, noalias should return void.
StephanTLavavej Jul 23, 2020
07e8d81
Cleanup: Use `const void* _Stop_at`, we don't write through this.
StephanTLavavej Jul 23, 2020
e59e076
Cleanups: while loop, and _Byte_length can just take `const void*`.
StephanTLavavej Jul 23, 2020
7ef5086
Copy-paste fix! Change 64 to 32.
StephanTLavavej Jul 23, 2020
5cd8c63
Suppress some overzealous static analysis warnings
CaseyCarter Jul 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions stl/inc/algorithm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,26 @@ _STL_DISABLE_CLANG_WARNINGS
#pragma push_macro("new")
#undef new

#if _USE_STD_VECTOR_ALGORITHMS
_EXTERN_C
// The "noalias" attribute tells the compiler optimizer that pointers going into these hand-vectorized algorithms
// won't be stored beyond the lifetime of the function, and that the function will only reference arrays denoted by
// those pointers. The optimizer also assumes in that case that a pointer parameter is not returned to the caller via
// the return value, so functions using "noalias" must usually return void. This attribute is valuable because these
// functions are in native code objects that the compiler cannot analyze. In the absence of the noalias attribute, the
// compiler has to assume that the denoted arrays are "globally address taken", and that any later calls to
// unanalyzable routines may modify those arrays.
__declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1(
const void* _First, const void* _Last, void* _Dest) noexcept;
__declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2(
const void* _First, const void* _Last, void* _Dest) noexcept;
__declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4(
const void* _First, const void* _Last, void* _Dest) noexcept;
__declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8(
const void* _First, const void* _Last, void* _Dest) noexcept;
_END_EXTERN_C
#endif // _USE_STD_VECTOR_ALGORITHMS

_STD_BEGIN
// COMMON SORT PARAMETERS
_INLINE_VAR constexpr int _ISORT_MAX = 32; // maximum size for insertion sort
Expand Down Expand Up @@ -4520,6 +4540,7 @@ namespace ranges {
constexpr bool _Allow_vectorization =
conjunction_v<_Is_trivially_swappable<_Elem>, negation<is_volatile<_Elem>>>;

#pragma warning(suppress : 6326) // Potential comparison of a constant with another constant
if constexpr (_Allow_vectorization && _Nx <= 8 && (_Nx & (_Nx - 1)) == 0) {
if (!_STD is_constant_evaluated()) {
_Elem* const _First_addr = _STD to_address(_First);
Expand Down Expand Up @@ -4584,6 +4605,37 @@ _CONSTEXPR20 _OutIt reverse_copy(_BidIt _First, _BidIt _Last, _OutIt _Dest) {
const auto _UFirst = _Get_unwrapped(_First);
auto _ULast = _Get_unwrapped(_Last);
auto _UDest = _Get_unwrapped_n(_Dest, _Idl_distance<_BidIt>(_UFirst, _ULast));

#if _HAS_IF_CONSTEXPR && _USE_STD_VECTOR_ALGORITHMS
using _Elem = remove_pointer_t<decltype(_UFirst)>;
using _DestElem = remove_pointer_t<decltype(_UDest)>;
constexpr bool _Allow_vectorization = conjunction_v<is_same<remove_const_t<_Elem>, remove_const_t<_DestElem>>,
is_pointer<decltype(_UFirst)>, is_trivially_copyable<_Elem>, negation<is_volatile<_Elem>>>;
constexpr size_t _Nx = sizeof(_Elem);

#pragma warning(suppress : 6326) // Potential comparison of a constant with another constant
if constexpr (_Allow_vectorization && _Nx <= 8 && (_Nx & (_Nx - 1)) == 0) {
#ifdef __cpp_lib_is_constant_evaluated
if (!_STD is_constant_evaluated())
#endif // __cpp_lib_is_constant_evaluated
{
if constexpr (_Nx == 1) {
__std_reverse_copy_trivially_copyable_1(_UFirst, _ULast, _UDest);
} else if constexpr (_Nx == 2) {
__std_reverse_copy_trivially_copyable_2(_UFirst, _ULast, _UDest);
} else if constexpr (_Nx == 4) {
__std_reverse_copy_trivially_copyable_4(_UFirst, _ULast, _UDest);
} else {
__std_reverse_copy_trivially_copyable_8(_UFirst, _ULast, _UDest);
}

_UDest += _ULast - _UFirst;
_Seek_wrapped(_Dest, _UDest);
return _Dest;
}
}
#endif // _HAS_IF_CONSTEXPR && _USE_STD_VECTOR_ALGORITHMS

for (; _UFirst != _ULast; ++_UDest) {
*_UDest = *--_ULast;
}
Expand Down
116 changes: 21 additions & 95 deletions stl/inc/xutility
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5481,116 +5481,42 @@ _NODISCARD _CONSTEXPR20 bool _Check_match_counts(
}

// FUNCTION TEMPLATE reverse
#if _HAS_IF_CONSTEXPR
template <class _BidIt>
_CONSTEXPR20 void reverse(const _BidIt _First, const _BidIt _Last) { // reverse elements in [_First, _Last)
_Adl_verify_range(_First, _Last);
auto _UFirst = _Get_unwrapped(_First);
auto _ULast = _Get_unwrapped(_Last);
#if _USE_STD_VECTOR_ALGORITHMS
#if _HAS_IF_CONSTEXPR && _USE_STD_VECTOR_ALGORITHMS
using _Elem = remove_pointer_t<decltype(_UFirst)>;
constexpr bool _Allow_vectorization =
conjunction_v<is_pointer<decltype(_UFirst)>, _Is_trivially_swappable<_Elem>, negation<is_volatile<_Elem>>>;
constexpr size_t _Nx = sizeof(_Elem);

if constexpr (_Allow_vectorization && sizeof(_Elem) == 1) {
#ifdef __cpp_lib_is_constant_evaluated
if (!_STD is_constant_evaluated())
#endif // __cpp_lib_is_constant_evaluated
{
__std_reverse_trivially_swappable_1(_UFirst, _ULast);
return;
}
} else if constexpr (_Allow_vectorization && sizeof(_Elem) == 2) {
#ifdef __cpp_lib_is_constant_evaluated
if (!_STD is_constant_evaluated())
#endif // __cpp_lib_is_constant_evaluated
{
__std_reverse_trivially_swappable_2(_UFirst, _ULast);
return;
}
} else if constexpr (_Allow_vectorization && sizeof(_Elem) == 4) {
#pragma warning(suppress : 6326) // Potential comparison of a constant with another constant
if constexpr (_Allow_vectorization && _Nx <= 8 && (_Nx & (_Nx - 1)) == 0) {
#ifdef __cpp_lib_is_constant_evaluated
if (!_STD is_constant_evaluated())
#endif // __cpp_lib_is_constant_evaluated
{
__std_reverse_trivially_swappable_4(_UFirst, _ULast);
return;
}
} else if constexpr (_Allow_vectorization && sizeof(_Elem) == 8) {
#ifdef __cpp_lib_is_constant_evaluated
if (!_STD is_constant_evaluated())
#endif // __cpp_lib_is_constant_evaluated
{
__std_reverse_trivially_swappable_8(_UFirst, _ULast);
if constexpr (_Nx == 1) {
__std_reverse_trivially_swappable_1(_UFirst, _ULast);
} else if constexpr (_Nx == 2) {
__std_reverse_trivially_swappable_2(_UFirst, _ULast);
} else if constexpr (_Nx == 4) {
__std_reverse_trivially_swappable_4(_UFirst, _ULast);
} else {
__std_reverse_trivially_swappable_8(_UFirst, _ULast);
}

return;
}
}
#endif // _USE_STD_VECTOR_ALGORITHMS
#endif // _HAS_IF_CONSTEXPR && _USE_STD_VECTOR_ALGORITHMS

for (; _UFirst != _ULast && _UFirst != --_ULast; ++_UFirst) {
_STD iter_swap(_UFirst, _ULast);
}
}
#else // ^^^ _HAS_IF_CONSTEXPR / !_HAS_IF_CONSTEXPR vvv
template <class _BidIt>
void _Reverse_unchecked1(_BidIt _First, _BidIt _Last, integral_constant<size_t, 0>) {
// reverse elements in [_First, _Last), general bidirectional iterators
for (; _First != _Last && _First != --_Last; ++_First) {
_STD iter_swap(_First, _Last);
}
}

#if _USE_STD_VECTOR_ALGORITHMS
template <class _BidIt>
void _Reverse_unchecked1(const _BidIt _First, const _BidIt _Last, integral_constant<size_t, 1>) {
// reverse elements in [_First, _Last), pointers to trivially swappable of size 1
__std_reverse_trivially_swappable_1(_First, _Last);
}

template <class _BidIt>
void _Reverse_unchecked1(const _BidIt _First, const _BidIt _Last, integral_constant<size_t, 2>) {
// reverse elements in [_First, _Last), pointers to trivially swappable of size 2
__std_reverse_trivially_swappable_2(_First, _Last);
}

template <class _BidIt>
void _Reverse_unchecked1(const _BidIt _First, const _BidIt _Last, integral_constant<size_t, 4>) {
// reverse elements in [_First, _Last), pointers to trivially swappable of size 4
__std_reverse_trivially_swappable_4(_First, _Last);
}

template <class _BidIt>
void _Reverse_unchecked1(const _BidIt _First, const _BidIt _Last, integral_constant<size_t, 8>) {
// reverse elements in [_First, _Last), pointers to trivially swappable of size 8
__std_reverse_trivially_swappable_8(_First, _Last);
}
#endif // _USE_STD_VECTOR_ALGORITHMS

template <class _BidIt>
void _Reverse_unchecked(const _BidIt _First, const _BidIt _Last) {
// reverse elements in [_First, _Last), choose optimization
#if _USE_STD_VECTOR_ALGORITHMS
using _Elem = remove_pointer_t<_BidIt>;
constexpr size_t _Opt =
is_pointer_v<_BidIt> //
&& _Is_trivially_swappable_v<_Elem> //
&& !is_volatile_v<_Elem> //
&& (sizeof(_Elem) == 1 || sizeof(_Elem) == 2 || sizeof(_Elem) == 4 || sizeof(_Elem) == 8)
? sizeof(_Elem)
: 0;
#else // ^^^ vectorize / no vectorize vvv
constexpr size_t _Opt = 0;
#endif // _USE_STD_VECTOR_ALGORITHMS
_Reverse_unchecked1(_First, _Last, integral_constant<size_t, _Opt>{});
}

template <class _BidIt>
void reverse(const _BidIt _First, const _BidIt _Last) {
// reverse elements in [_First, _Last)
_Adl_verify_range(_First, _Last);
_Reverse_unchecked(_Get_unwrapped(_First), _Get_unwrapped(_Last));
}
#endif // _HAS_IF_CONSTEXPR

#if _HAS_CXX17
template <class _ExPo, class _BidIt, _Enable_if_execution_policy_t<_ExPo> = 0>
Expand Down Expand Up @@ -5694,19 +5620,19 @@ _FwdIt _Rotate_unchecked1(_FwdIt _First, _FwdIt _Mid, _FwdIt _Last, forward_iter
template <class _BidIt>
_BidIt _Rotate_unchecked1(_BidIt _First, _BidIt _Mid, _BidIt _Last, bidirectional_iterator_tag) {
// rotate [_First, _Last) left by distance(_First, _Mid) positions, bidirectional iterators
_Reverse_unchecked(_First, _Mid);
_Reverse_unchecked(_Mid, _Last);
_STD reverse(_First, _Mid);
_STD reverse(_Mid, _Last);
auto _Tmp = _Reverse_until_sentinel_unchecked(_First, _Mid, _Last);
_Reverse_unchecked(_Tmp.first, _Tmp.second);
_STD reverse(_Tmp.first, _Tmp.second);
return _Mid != _Tmp.first ? _Tmp.first : _Tmp.second;
}

template <class _RanIt>
_RanIt _Rotate_unchecked1(_RanIt _First, _RanIt _Mid, _RanIt _Last, random_access_iterator_tag) {
// rotate [_First, _Last) left by distance(_First, _Mid) positions, random-access iterators
_Reverse_unchecked(_First, _Mid);
_Reverse_unchecked(_Mid, _Last);
_Reverse_unchecked(_First, _Last);
_STD reverse(_First, _Mid);
_STD reverse(_Mid, _Last);
_STD reverse(_First, _Last);
return _First + (_Last - _Mid);
}

Expand Down
Loading