Faster IndexOf for substrings

THIRD-PARTY-NOTICES.TXT

@@ -697,6 +697,35 @@ License for fastmod (https://github.com/lemire/fastmod) and ibm-fpgen (https://g
    See the License for the specific language governing permissions and
    limitations under the License.
 
+License for sse4-strstr (https://github.com/WojciechMula/sse4-strstr)
+--------------------------------------
+
+   Copyright (c) 2008-2016, Wojciech Muła
+   All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+   IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+   TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+   PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 License notice for The C++ REST SDK
 -----------------------------------
 

src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs

@@ -580,12 +580,25 @@ ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T> value) where T : IEquatable<T>
         {
-            if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
-                return SpanHelpers.LastIndexOf(
-                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                    span.Length,
-                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
-                    value.Length);
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOf(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        span.Length,
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
+                        value.Length);
+                }
+                if (Unsafe.SizeOf<T>() == sizeof(char))
+                {
+                    return SpanHelpers.LastIndexOf(
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
+                        span.Length,
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
+                        value.Length);
+                }
+            }
 
             return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
         }

src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs

@@ -708,5 +708,25 @@ public static nuint RotateRight(nuint value, int offset)
             return (nuint)RotateRight((uint)value, offset);
 #endif
         }
+
+        /// <summary>
+        /// Reset the lowest significant bit in the given value
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static uint ResetLowestSetBit(uint value)
+        {
+            // It's lowered to BLSR on x86
+            return value & (value - 1);
+        }
+
+        /// <summary>
+        /// Reset specific bit in the given value
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static uint ResetBit(uint value, int bitPos)
+        {
+            // TODO: Recognize BTR on x86 and LSL+BIC on ARM
+            return value & ~(uint)(1 << bitPos);
+        }
     }
 }

src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs

@@ -22,12 +22,26 @@ public static int IndexOf(ref byte searchSpace, int searchSpaceLength, ref byte
             if (valueLength == 0)
                 return 0;  // A zero-length sequence is always treated as "found" at the start of the search space.
 
+            int valueTailLength = valueLength - 1;
+
+            if (valueTailLength == 0)
+            {
+                // for single-byte values use plain IndexOf
+                return IndexOf(ref searchSpace, value, searchSpaceLength);
+            }
+
             byte valueHead = value;
             ref byte valueTail = ref Unsafe.Add(ref value, 1);
-            int valueTailLength = valueLength - 1;
+            int offset = 0;
+            nuint valueTailNLength = (nuint)(uint)valueTailLength;
+
+            if (Vector128.IsHardwareAccelerated && searchSpaceLength - valueTailLength >= Vector128<byte>.Count)
+            {
+                goto SEARCH_TWO_BYTES;
+            }
+
             int remainingSearchSpaceLength = searchSpaceLength - valueTailLength;
 
-            int offset = 0;
             while (remainingSearchSpaceLength > 0)
             {
                 // Do a quick search for the first element of "value".
@@ -42,13 +56,119 @@ public static int IndexOf(ref byte searchSpace, int searchSpaceLength, ref byte
                     break;  // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there.
 
                 // Found the first element of "value". See if the tail matches.
-                if (SequenceEqual(ref Unsafe.Add(ref searchSpace, offset + 1), ref valueTail, (nuint)valueTailLength))  // The (nuint)-cast is necessary to pick the correct overload
+                if (SequenceEqual(ref Unsafe.Add(ref searchSpace, offset + 1), ref valueTail, valueTailNLength))  // The (nuint)-cast is necessary to pick the correct overload
                     return offset;  // The tail matched. Return a successful find.
 
                 remainingSearchSpaceLength--;
                 offset++;
             }
             return -1;
+
+            // Based on http://0x80.pl/articles/simd-strfind.html#algorithm-1-generic-simd "Algorithm 1: Generic SIMD" by Wojciech Muła
+            // Some details about the implementation can also be found in https://github.com/dotnet/runtime/pull/63285
+        SEARCH_TWO_BYTES:
+            if (Avx2.IsSupported && searchSpaceLength - valueTailLength >= Vector256<byte>.Count)
+            {
+                // Find the last unique (which is not equal to ch1) byte
+                // the algorithm is fine if both are equal, just a little bit less efficient
+                byte ch2Val = Unsafe.Add(ref value, valueTailLength);
+                int ch1ch2Distance = valueTailLength;
+                while (ch2Val == value && ch1ch2Distance > 1)
+                    ch2Val = Unsafe.Add(ref value, --ch1ch2Distance);
+
+                Vector256<byte> ch1 = Vector256.Create(value);
+                Vector256<byte> ch2 = Vector256.Create(ch2Val);
+
+                do
+                {
+                    Vector256<byte> cmpCh1 = Vector256.Equals(ch1, Vector256.LoadUnsafe(ref searchSpace, (nuint)offset));
+                    Vector256<byte> cmpCh2 = Vector256.Equals(ch2, Vector256.LoadUnsafe(ref searchSpace, (nuint)(offset + ch1ch2Distance)));
+                    Vector256<byte> cmpAnd = (cmpCh1 & cmpCh2).AsByte();
+
+                    // Early out: cmpAnd is all zeros
+                    if (cmpAnd != Vector256<byte>.Zero)
+                    {
+                        uint mask = cmpAnd.ExtractMostSignificantBits();
+                        do
+                        {
+                            int bitPos = BitOperations.TrailingZeroCount(mask);
+                            if (valueTailNLength == 1 || // we already matched two bytes
+                                SequenceEqual(
+                                    ref Unsafe.Add(ref searchSpace, offset + bitPos + 1),
+                                    ref valueTail,
+                                    valueTailNLength))
+                            {
+                                return offset + bitPos;
+                            }
+                            // Clear the lowest set bit
+                            mask = BitOperations.ResetLowestSetBit(mask);
+                        } while (mask != 0);
+                    }
+
+                    offset += Vector256<byte>.Count;
+
+                    if (offset + valueTailLength == searchSpaceLength)
+                        return -1;
+
+                    // Overlap with the current chunk if there is not enough room for the next one
+                    if (offset + valueTailLength + Vector256<byte>.Count > searchSpaceLength)
+                        offset = searchSpaceLength - valueTailLength - Vector256<byte>.Count;
+
+                } while (true);
+            }
+
+            if (Vector128.IsHardwareAccelerated)
+            {
+                // Find the last unique (which is not equal to ch1) byte
+                // the algorithm is fine if both are equal, just a little bit less efficient
+                byte ch2Val = Unsafe.Add(ref value, valueTailLength);
+                int ch1ch2Distance = valueTailLength;
+                while (ch2Val == value && ch1ch2Distance > 1)
+                    ch2Val = Unsafe.Add(ref value, --ch1ch2Distance);
+
+                Vector128<byte> ch1 = Vector128.Create(value);
+                Vector128<byte> ch2 = Vector128.Create(ch2Val);
+
+                do
+                {
+                    Vector128<byte> cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref searchSpace, (nuint)offset));
+                    Vector128<byte> cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref searchSpace, (nuint)(offset + ch1ch2Distance)));
+                    Vector128<byte> cmpAnd = (cmpCh1 & cmpCh2).AsByte();
+
+                    // Early out: cmpAnd is all zeros
+                    // it's especially important for ARM where ExtractMostSignificantBits is not cheap
+                    if (cmpAnd != Vector128<byte>.Zero)
+                    {
+                        uint mask = cmpAnd.ExtractMostSignificantBits();
+                        do
+                        {
+                            int bitPos = BitOperations.TrailingZeroCount(mask);
+                            if (valueTailNLength == 1 || // we already matched two bytes
+                                SequenceEqual(
+                                    ref Unsafe.Add(ref searchSpace, offset + bitPos + 1),
+                                    ref valueTail,
+                                    valueTailNLength))
+                            {
+                                return offset + bitPos;
+                            }
+                            // Clear the lowest set bit
+                            mask = BitOperations.ResetLowestSetBit(mask);
+                        } while (mask != 0);
+                    }
+                    offset += Vector128<byte>.Count;
+
+                    if (offset + valueTailLength == searchSpaceLength)
+                        return -1;
+
+                    // Overlap with the current chunk if there is not enough room for the next one
+                    if (offset + valueTailLength + Vector128<byte>.Count > searchSpaceLength)
+                        offset = searchSpaceLength - valueTailLength - Vector128<byte>.Count;
+
+                } while (true);
+            }
+
+            Debug.Fail("Unreachable");
+            return -1;
         }
 
         // Adapted from IndexOf(...)
@@ -416,11 +536,24 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b
             if (valueLength == 0)
                 return searchSpaceLength;  // A zero-length sequence is always treated as "found" at the end of the search space.
 
-            byte valueHead = value;
-            ref byte valueTail = ref Unsafe.Add(ref value, 1);
             int valueTailLength = valueLength - 1;
 
+            if (valueTailLength == 0)
+            {
+                // for single-byte values use plain LastIndexOf
+                return LastIndexOf(ref searchSpace, value, searchSpaceLength);
+            }
+
+            byte valueHead = value;
+            ref byte valueTail = ref Unsafe.Add(ref value, 1);
             int offset = 0;
+            nuint valueTailNLength = (nuint)(uint)valueTailLength;
+
+            if (Vector128.IsHardwareAccelerated && searchSpaceLength - valueTailLength >= Vector128<byte>.Count)
+            {
+                goto SEARCH_TWO_BYTES;
+            }
+
             while (true)
             {
                 Debug.Assert(0 <= offset && offset <= searchSpaceLength); // Ensures no deceptive underflows in the computation of "remainingSearchSpaceLength".
@@ -434,12 +567,120 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b
                     break;
 
                 // Found the first element of "value". See if the tail matches.
-                if (SequenceEqual(ref Unsafe.Add(ref searchSpace, relativeIndex + 1), ref valueTail, (nuint)(uint)valueTailLength))  // The (nunit)-cast is necessary to pick the correct overload
+                if (SequenceEqual(ref Unsafe.Add(ref searchSpace, relativeIndex + 1), ref valueTail, valueTailNLength))  // The (nunit)-cast is necessary to pick the correct overload
                     return relativeIndex;  // The tail matched. Return a successful find.
 
                 offset += remainingSearchSpaceLength - relativeIndex;
             }
             return -1;
+
+            // Based on http://0x80.pl/articles/simd-strfind.html#algorithm-1-generic-simd "Algorithm 1: Generic SIMD" by Wojciech Muła
+            // Some details about the implementation can also be found in https://github.com/dotnet/runtime/pull/63285
+        SEARCH_TWO_BYTES:
+            if (Avx2.IsSupported && searchSpaceLength - valueTailLength >= Vector256<byte>.Count)
+            {
+                offset = searchSpaceLength - valueTailLength - Vector256<byte>.Count;
+
+                // Find the last unique (which is not equal to ch1) byte
+                // the algorithm is fine if both are equal, just a little bit less efficient
+                byte ch2Val = Unsafe.Add(ref value, valueTailLength);
+                int ch1ch2Distance = valueTailLength;
+                while (ch2Val == value && ch1ch2Distance > 1)
+                    ch2Val = Unsafe.Add(ref value, --ch1ch2Distance);
+
+                Vector256<byte> ch1 = Vector256.Create(value);
+                Vector256<byte> ch2 = Vector256.Create(ch2Val);
+
+                do
+                {
+                    Vector256<byte> cmpCh1 = Vector256.Equals(ch1, Vector256.LoadUnsafe(ref searchSpace, (nuint)offset));
+                    Vector256<byte> cmpCh2 = Vector256.Equals(ch2, Vector256.LoadUnsafe(ref searchSpace, (nuint)(offset + ch1ch2Distance)));
+                    Vector256<byte> cmpAnd = (cmpCh1 & cmpCh2).AsByte();
+
+                    // Early out: cmpAnd is all zeros
+                    if (cmpAnd != Vector256<byte>.Zero)
+                    {
+                        uint mask = cmpAnd.ExtractMostSignificantBits();
+                        do
+                        {
+                            // unlike IndexOf, here we use LZCNT to process matches starting from the end
+                            int bitPos = 31 - BitOperations.LeadingZeroCount(mask);
+                            if (valueTailNLength == 1 || // we already matched two bytes
+                                SequenceEqual(
+                                    ref Unsafe.Add(ref searchSpace, offset + bitPos + 1),
+                                    ref valueTail,
+                                    valueTailNLength))
+                            {
+                                return bitPos + offset;
+                            }
+                            // Clear the highest set bit.
+                            mask = BitOperations.ResetBit(mask, bitPos);
+                        } while (mask != 0);
+                    }
+
+                    offset -= Vector256<byte>.Count;
+                    if (offset == -Vector256<byte>.Count)
+                        return -1;
+                    // Overlap with the current chunk if there is not enough room for the next one
+                    if (offset < 0)
+                        offset = 0;
+
+                } while (true);
+            }
+            if (Vector128.IsHardwareAccelerated)
+            {
+                offset = searchSpaceLength - valueTailLength - Vector128<byte>.Count;
+
+                // Find the last unique (which is not equal to ch1) byte
+                // the algorithm is fine if both are equal, just a little bit less efficient
+                byte ch2Val = Unsafe.Add(ref value, valueTailLength);
+                int ch1ch2Distance = valueTailLength;
+                while (ch2Val == value && ch1ch2Distance > 1)
+                    ch2Val = Unsafe.Add(ref value, --ch1ch2Distance);
+
+                Vector128<byte> ch1 = Vector128.Create(value);
+                Vector128<byte> ch2 = Vector128.Create(ch2Val);
+
+                do
+                {
+                    Vector128<byte> cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref searchSpace, (nuint)offset));
+                    Vector128<byte> cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref searchSpace, (nuint)(offset + ch1ch2Distance)));
+                    Vector128<byte> cmpAnd = (cmpCh1 & cmpCh2).AsByte();
+
+                    // Early out: cmpAnd is all zeros
+                    // it's especially important for ARM where ExtractMostSignificantBits is not cheap
+                    if (cmpAnd != Vector128<byte>.Zero)
+                    {
+                        uint mask = cmpAnd.ExtractMostSignificantBits();
+                        do
+                        {
+                            // unlike IndexOf, here we use LZCNT to process matches starting from the end
+                            int bitPos = 31 - BitOperations.LeadingZeroCount(mask);
+                            if (valueTailNLength == 1 || // we already matched two bytes
+                                SequenceEqual(
+                                    ref Unsafe.Add(ref searchSpace, offset + bitPos + 1),
+                                    ref valueTail,
+                                    valueTailNLength))
+                            {
+                                return bitPos + offset;
+                            }
+                            // Clear the highest set bit.
+                            mask = BitOperations.ResetBit(mask, bitPos);
+                        } while (mask != 0);
+                    }
+
+                    offset -= Vector128<byte>.Count;
+                    if (offset == -Vector128<byte>.Count)
+                        return -1;
+                    // Overlap with the current chunk if there is not enough room for the next one
+                    if (offset < 0)
+                        offset = 0;
+
+                } while (true);
+            }
+
+            Debug.Fail("Unreachable");
+            return -1;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]