diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 52524425b7674c..b5cdd6ead448cd 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -425,20 +425,21 @@ - + + + - @@ -1031,8 +1032,8 @@ - + diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 0ed1bcc8c5bdb7..f5157b0cbba8ad 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -339,8 +339,8 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan source, ReadOnly { // Do a quick search for the first element of "value". int relativeIndex = isLetter ? - SpanHelpers.PackedIndexOfIsSupported - ? SpanHelpers.PackedIndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) + PackedSpanHelpers.PackedIndexOfIsSupported + ? PackedSpanHelpers.PackedIndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) : SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength); if (relativeIndex < 0) diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs index 7579de199e77b8..8461c8b2b50c52 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny1CharValue.cs @@ -23,7 +23,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) + ? PackedSpanHelpers.PackedIndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) : SpanHelpers.NonPackedIndexOfValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), @@ -32,7 +32,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) + ? PackedSpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) : SpanHelpers.NonPackedIndexOfValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs index 9ca43deb7f5225..8560999b3d4bfb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny2CharValues.cs @@ -14,7 +14,7 @@ internal sealed class IndexOfAny2CharValue : IndexOfAnyValues< public IndexOfAny2CharValue(char value0, char value1) => (_e0, _e1) = (value0, value1); - internal override char[] GetValues() => new[] { _e0 }; + internal override char[] GetValues() => new[] { _e0, _e1 }; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override bool ContainsCore(char value) => @@ -23,7 +23,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + ? PackedSpanHelpers.PackedIndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) : SpanHelpers.NonPackedIndexOfAnyValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), @@ -33,7 +33,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) + ? PackedSpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) : SpanHelpers.NonPackedIndexOfAnyValueType>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _e0), diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3Values.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3ByteValues.cs similarity index 63% rename from src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3Values.cs rename to src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3ByteValues.cs index f94509b19f6d6a..ad569e18589ac3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3Values.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3ByteValues.cs @@ -2,42 +2,40 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using System.Numerics; using System.Runtime.CompilerServices; namespace System.Buffers { - internal sealed class IndexOfAny3Values : IndexOfAnyValues - where T : struct, INumber + internal sealed class IndexOfAny3ByteValues : IndexOfAnyValues { - private readonly T _e0, _e1, _e2; + private readonly byte _e0, _e1, _e2; - public IndexOfAny3Values(ReadOnlySpan values) + public IndexOfAny3ByteValues(ReadOnlySpan values) { Debug.Assert(values.Length == 3); (_e0, _e1, _e2) = (values[0], values[1], values[2]); } - internal override T[] GetValues() => new[] { _e0, _e1, _e2 }; + internal override byte[] GetValues() => new[] { _e0, _e1, _e2 }; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(T value) => + internal override bool ContainsCore(byte value) => value == _e0 || value == _e1 || value == _e2; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => + internal override int IndexOfAny(ReadOnlySpan span) => span.IndexOfAny(_e0, _e1, _e2); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => + internal override int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(_e0, _e1, _e2); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => + internal override int LastIndexOfAny(ReadOnlySpan span) => span.LastIndexOfAny(_e0, _e1, _e2); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.LastIndexOfAnyExcept(_e0, _e1, _e2); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs new file mode 100644 index 00000000000000..519fe27245def9 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAny3CharValues.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + internal sealed class IndexOfAny3CharValue : IndexOfAnyValues + where TShouldUsePacked : struct, IndexOfAnyValues.IRuntimeConst + { + private char _e0, _e1, _e2; + + public IndexOfAny3CharValue(char value0, char value1, char value2) => + (_e0, _e1, _e2) = (value0, value1, value2); + + internal override char[] GetValues() => new[] { _e0, _e1, _e2 }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0 || value == _e1 || value == _e2; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.PackedIndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + Unsafe.As(ref _e2), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + TShouldUsePacked.Value + ? PackedSpanHelpers.PackedIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) + : SpanHelpers.NonPackedIndexOfAnyValueType>( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), + Unsafe.As(ref _e0), + Unsafe.As(ref _e1), + Unsafe.As(ref _e2), + span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_e0, _e1, _e2); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0, _e1, _e2); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs index 11fe35bd9d8657..961de773e25796 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyCharValuesInRange.cs @@ -39,7 +39,7 @@ internal override bool ContainsCore(char value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAnyInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + ? PackedSpanHelpers.PackedIndexOfAnyInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _lowInclusive), @@ -49,7 +49,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(span)), [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAnyExcept(ReadOnlySpan span) => TShouldUsePacked.Value - ? SpanHelpers.PackedIndexOfAnyExceptInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) + ? PackedSpanHelpers.PackedIndexOfAnyExceptInRange(ref MemoryMarshal.GetReference(span), _lowInclusive, _rangeInclusive, span.Length) : SpanHelpers.NonPackedIndexOfAnyInRangeUnsignedNumber>( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), Unsafe.As(ref _lowInclusive), diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs index bccdd87d214286..24c62bd92b9695 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyValues.cs @@ -48,7 +48,7 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) return values.Length switch { 2 => new IndexOfAny2ByteValues(values), - 3 => new IndexOfAny3Values(values), + 3 => new IndexOfAny3ByteValues(values), 4 => new IndexOfAny4Values(values), _ => new IndexOfAny5Values(values), }; @@ -80,7 +80,7 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) if (values.Length == 1) { char value = values[0]; - return SpanHelpers.CanUsePackedIndexOf(value) + return PackedSpanHelpers.CanUsePackedIndexOf(value) ? new IndexOfAny1CharValue(value) : new IndexOfAny1CharValue(value); } @@ -95,14 +95,19 @@ public static IndexOfAnyValues Create(ReadOnlySpan values) { char value0 = values[0]; char value1 = values[1]; - return SpanHelpers.CanUsePackedIndexOf(value0) && SpanHelpers.CanUsePackedIndexOf(value1) + return PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) ? new IndexOfAny2CharValue(value0, value1) : new IndexOfAny2CharValue(value0, value1); } if (values.Length == 3) { - return new IndexOfAny3Values(values); + char value0 = values[0]; + char value1 = values[1]; + char value2 = values[2]; + return PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) && PackedSpanHelpers.CanUsePackedIndexOf(value2) + ? new IndexOfAny3CharValue(value0, value1, value2) + : new IndexOfAny3CharValue(value0, value1, value2); } // IndexOfAnyAsciiSearcher for chars is slower than IndexOfAny3Values, but faster than IndexOfAny4Values @@ -180,7 +185,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), } Debug.Assert(typeof(T) == typeof(char)); - return (IndexOfAnyValues)(object)(SpanHelpers.CanUsePackedIndexOf(min) && SpanHelpers.CanUsePackedIndexOf(max) + return (IndexOfAnyValues)(object)(PackedSpanHelpers.CanUsePackedIndexOf(min) && PackedSpanHelpers.CanUsePackedIndexOf(max) ? new IndexOfAnyCharValuesInRange(*(char*)&min, *(char*)&max) : new IndexOfAnyCharValuesInRange(*(char*)&min, *(char*)&max)); } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs similarity index 61% rename from src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs rename to src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index 02d76e66077cbf..8fce1de5e3216b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -9,19 +9,23 @@ using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 + #pragma warning disable 8500 // sizeof of managed types namespace System { - internal static partial class SpanHelpers // .Char.Packed + // This is a separate class instead of 'partial SpanHelpers' to hide the private helpers + // included in this file which are specific to the packed implementation. + internal static partial class PackedSpanHelpers { - internal static bool PackedIndexOfIsSupported => Sse2.IsSupported || AdvSimd.IsSupported; + public static bool PackedIndexOfIsSupported => Sse2.IsSupported || AdvSimd.IsSupported; // Not all values can benefit from packing the searchSpace. See comments in PackSources below. // On X86, the values must be in the [1, 254] range. // On ARM, the values must be in the [0, 254] range. [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe bool CanUsePackedIndexOf(T value) => + public static unsafe bool CanUsePackedIndexOf(T value) => PackedIndexOfIsSupported && RuntimeHelpers.IsBitwiseEquatable() && sizeof(T) == sizeof(ushort) && @@ -30,31 +34,181 @@ internal static unsafe bool CanUsePackedIndexOf(T value) => : *(ushort*)&value < 255u); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOf(ref char searchSpace, char value, int length) => - PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + public static int PackedIndexOf(ref char searchSpace, char value, int length) => + PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAnyExcept(ref char searchSpace, char value, int length) => - PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + public static int PackedIndexOfAnyExcept(ref char searchSpace, char value, int length) => + PackedIndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAny(ref char searchSpace, char value0, char value1, int length) => - PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + public static int PackedIndexOfAny(ref char searchSpace, char value0, char value1, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => - PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + public static int PackedIndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => - PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + public static int PackedIndexOfAny(ref char searchSpace, char value0, char value1, char value2, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int PackedIndexOfAnyExceptInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => - PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + public static int PackedIndexOfAnyExcept(ref char searchSpace, char value0, char value1, char value2, int length) => + PackedIndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int PackedIndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int PackedIndexOfAnyExceptInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => + PackedIndexOfAnyInRange>(ref Unsafe.As(ref searchSpace), (short)lowInclusive, (short)rangeInclusive, length); + + public static bool PackedContains(ref short searchSpace, short value, int length) + { + Debug.Assert(CanUsePackedIndexOf(value)); + + if (length < Vector128.Count) + { + nuint offset = 0; + + if (length >= 4) + { + length -= 4; + + if (searchSpace == value || + Unsafe.Add(ref searchSpace, 1) == value || + Unsafe.Add(ref searchSpace, 2) == value || + Unsafe.Add(ref searchSpace, 3) == value) + { + return true; + } + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + if (Unsafe.Add(ref searchSpace, offset) == value) + { + return true; + } + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue = Vector256.Create((byte)value); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + + if (result != Vector256.Zero) + { + return true; + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we're only interested in whether any value matched. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue, packedSource); + + if (result != Vector256.Zero) + { + return true; + } + } + } + else + { + Vector128 packedValue = Vector128.Create((byte)value); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + + if (result != Vector128.Zero) + { + return true; + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we're only interested in whether any value matched. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue, packedSource); + + if (result != Vector128.Zero) + { + return true; + } + } + } + } + + return false; + } private static int PackedIndexOf(ref short searchSpace, short value, int length) - where TNegator : struct, INegator + where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(value)); @@ -195,7 +349,7 @@ private static int PackedIndexOf(ref short searchSpace, short value, i } private static int PackedIndexOfAny(ref short searchSpace, short value0, short value1, int length) - where TNegator : struct, INegator + where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(value0)); Debug.Assert(CanUsePackedIndexOf(value1)); @@ -344,8 +498,161 @@ private static int PackedIndexOfAny(ref short searchSpace, short value return -1; } + private static int PackedIndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) + where TNegator : struct, SpanHelpers.INegator + { + Debug.Assert(CanUsePackedIndexOf(value0)); + Debug.Assert(CanUsePackedIndexOf(value1)); + Debug.Assert(CanUsePackedIndexOf(value2)); + + if (length < Vector128.Count) + { + nuint offset = 0; + short lookUp; + + if (length >= 4) + { + length -= 4; + + lookUp = searchSpace; + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 0; + lookUp = Unsafe.Add(ref searchSpace, 1); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 1; + lookUp = Unsafe.Add(ref searchSpace, 2); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 2; + lookUp = Unsafe.Add(ref searchSpace, 3); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 3; + + offset = 4; + } + + while (length > 0) + { + length -= 1; + + lookUp = Unsafe.Add(ref searchSpace, offset); + if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; + + offset += 1; + } + } + else + { + ref short currentSearchSpace = ref searchSpace; + + if (Avx2.IsSupported && length > Vector256.Count) + { + Vector256 packedValue0 = Vector256.Create((byte)value0); + Vector256 packedValue1 = Vector256.Create((byte)value1); + Vector256 packedValue2 = Vector256.Create((byte)value2); + + if (length > 2 * Vector256.Count) + { + // Process the input in chunks of 32 characters (2 * Vector256). + // If the input length is a multiple of 32, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector256.Count)); + + do + { + Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); + Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector256.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-32 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); + Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector256 packedSource = PackSources(source0, source1); + Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector256.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + else + { + Vector128 packedValue0 = Vector128.Create((byte)value0); + Vector128 packedValue1 = Vector128.Create((byte)value1); + Vector128 packedValue2 = Vector128.Create((byte)value2); + + if (!Avx2.IsSupported && length > 2 * Vector128.Count) + { + // Process the input in chunks of 16 characters (2 * Vector128). + // If the input length is a multiple of 16, don't consume the last 16 characters in this loop. + // Let the fallback below handle it instead. This is why the condition is + // ">" instead of ">=" above, and why "IsAddressLessThan" is used instead of "!IsAddressGreaterThan". + ref short twoVectorsAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - (2 * Vector128.Count)); + + do + { + Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); + Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, result); + } + + currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, 2 * Vector128.Count); + } + while (Unsafe.IsAddressLessThan(ref currentSearchSpace, ref twoVectorsAwayFromEnd)); + } + + // We have 1-16 characters remaining. Process the first and last vector in the search space. + // They may overlap, but we'll handle that in the index calculation if we do get a match. + { + ref short oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + + ref short firstVector = ref Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd) + ? ref oneVectorAwayFromEnd + : ref currentSearchSpace; + + Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); + Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); + Vector128 packedSource = PackSources(source0, source1); + Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); + result = NegateIfNeeded(result); + + if (result != Vector128.Zero) + { + return ComputeFirstIndexOverlapped(ref searchSpace, ref firstVector, ref oneVectorAwayFromEnd, result); + } + } + } + } + + return -1; + } + private static int PackedIndexOfAnyInRange(ref short searchSpace, short lowInclusive, short rangeInclusive, int length) - where TNegator : struct, INegator + where TNegator : struct, SpanHelpers.INegator { Debug.Assert(CanUsePackedIndexOf(lowInclusive)); Debug.Assert(CanUsePackedIndexOf((short)(lowInclusive + rangeInclusive))); @@ -504,13 +811,13 @@ private static Vector128 PackSources(Vector128 source0, Vector128 NegateIfNeeded(Vector128 result) - where TNegator : struct, INegator => - typeof(TNegator) == typeof(DontNegate) ? result : ~result; + where TNegator : struct, SpanHelpers.INegator => + typeof(TNegator) == typeof(SpanHelpers.DontNegate) ? result : ~result; [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector256 NegateIfNeeded(Vector256 result) - where TNegator : struct, INegator => - typeof(TNegator) == typeof(DontNegate) ? result : ~result; + where TNegator : struct, SpanHelpers.INegator => + typeof(TNegator) == typeof(SpanHelpers.DontNegate) ? result : ~result; [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeFirstIndex(ref short searchSpace, ref short current, Vector128 equals) @@ -528,7 +835,6 @@ private static int ComputeFirstIndex(ref short searchSpace, ref short current, V return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / sizeof(short)); } -#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ComputeFirstIndexOverlapped(ref short searchSpace, ref short current0, ref short current1, Vector128 equals) { @@ -556,7 +862,6 @@ private static int ComputeFirstIndexOverlapped(ref short searchSpace, ref short } return offsetInVector + (int)(Unsafe.ByteOffset(ref searchSpace, ref current0) / sizeof(short)); } -#pragma warning restore IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint FixUpPackedVector256Mask(uint mask) diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index bbfdcaf120e524..4ba9a82643ce22 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1303,8 +1303,19 @@ public static int SequenceCompareTo(ref T first, int firstLength, ref T secon return firstLength.CompareTo(secondLength); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe bool ContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber + { + if (PackedSpanHelpers.CanUsePackedIndexOf(value)) + { + return PackedSpanHelpers.PackedContains(ref Unsafe.As(ref searchSpace), *(short*)&value, length); + } + + return NonPackedContainsValueType(ref searchSpace, value, length); + } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] - internal static bool ContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber + internal static bool NonPackedContainsValueType(ref T searchSpace, T value, int length) where T : struct, INumber { Debug.Assert(length >= 0, "Expected non-negative length"); Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types"); @@ -1439,11 +1450,11 @@ private static unsafe int IndexOfValueType(ref TValue searchSp where TValue : struct, INumber where TNegator : struct, INegator { - if (CanUsePackedIndexOf(value)) + if (PackedSpanHelpers.CanUsePackedIndexOf(value)) { return typeof(TNegator) == typeof(DontNegate) - ? PackedIndexOf(ref Unsafe.As(ref searchSpace), *(char*)&value, length) - : PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value, length); + ? PackedSpanHelpers.PackedIndexOf(ref Unsafe.As(ref searchSpace), *(char*)&value, length) + : PackedSpanHelpers.PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value, length); } return NonPackedIndexOfValueType(ref searchSpace, value, length); @@ -1581,11 +1592,11 @@ private static unsafe int IndexOfAnyValueType(ref TValue searc where TValue : struct, INumber where TNegator : struct, INegator { - if (CanUsePackedIndexOf(value0) && CanUsePackedIndexOf(value1)) + if (PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1)) { return typeof(TNegator) == typeof(DontNegate) - ? PackedIndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) - : PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); + ? PackedSpanHelpers.PackedIndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) + : PackedSpanHelpers.PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); } return NonPackedIndexOfAnyValueType(ref searchSpace, value0, value1, length); @@ -1738,8 +1749,23 @@ internal static int IndexOfAnyExceptValueType(ref T searchSpace, T value0, T => IndexOfAnyValueType>(ref searchSpace, value0, value1, value2, length); #endif + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) + where TValue : struct, INumber + where TNegator : struct, INegator + { + if (PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) && PackedSpanHelpers.CanUsePackedIndexOf(value2)) + { + return typeof(TNegator) == typeof(DontNegate) + ? PackedSpanHelpers.PackedIndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length) + : PackedSpanHelpers.PackedIndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, *(char*)&value2, length); + } + + return NonPackedIndexOfAnyValueType(ref searchSpace, value0, value1, value2, length); + } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private static int IndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) + internal static int NonPackedIndexOfAnyValueType(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length) where TValue : struct, INumber where TNegator : struct, INegator { @@ -2945,15 +2971,15 @@ private static unsafe int IndexOfAnyInRangeUnsignedNumber(ref T sea where T : struct, IUnsignedNumber, IComparisonOperators where TNegator : struct, INegator { - if (CanUsePackedIndexOf(lowInclusive) && CanUsePackedIndexOf(highInclusive) && highInclusive >= lowInclusive) + if (PackedSpanHelpers.CanUsePackedIndexOf(lowInclusive) && PackedSpanHelpers.CanUsePackedIndexOf(highInclusive) && highInclusive >= lowInclusive) { ref char charSearchSpace = ref Unsafe.As(ref searchSpace); char charLowInclusive = *(char*)&lowInclusive; char charRange = (char)(*(char*)&highInclusive - charLowInclusive); return typeof(TNegator) == typeof(DontNegate) - ? PackedIndexOfAnyInRange(ref charSearchSpace, charLowInclusive, charRange, length) - : PackedIndexOfAnyExceptInRange(ref charSearchSpace, charLowInclusive, charRange, length); + ? PackedSpanHelpers.PackedIndexOfAnyInRange(ref charSearchSpace, charLowInclusive, charRange, length) + : PackedSpanHelpers.PackedIndexOfAnyExceptInRange(ref charSearchSpace, charLowInclusive, charRange, length); } return NonPackedIndexOfAnyInRangeUnsignedNumber(ref searchSpace, lowInclusive, highInclusive, length); diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs index c84b07a3c9f755..feda6cc6488945 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs @@ -78,8 +78,8 @@ private int IndexOfCharOrdinalIgnoreCase(char value) { char valueUc = (char)(value | 0x20); char valueLc = (char)(value & ~0x20); - return SpanHelpers.PackedIndexOfIsSupported - ? SpanHelpers.PackedIndexOfAny(ref _firstChar, valueLc, valueUc, Length) + return PackedSpanHelpers.PackedIndexOfIsSupported + ? PackedSpanHelpers.PackedIndexOfAny(ref _firstChar, valueLc, valueUc, Length) : SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); }