Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Mono] Remove Vector<T> fallbacks #88101

Merged
merged 1 commit into from
Jun 28, 2023
Merged

[Mono] Remove Vector<T> fallbacks #88101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 0 additions & 145 deletions src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,12 +462,6 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace)
// Avx2 branch also operates on Sse2 sizes, so check is combined.
lengthToExamine = UnalignedCountVector128(searchSpace);
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
lengthToExamine = UnalignedCountVector(searchSpace);
}
#endif

SequentialScan:
while (lengthToExamine >= 8)
Expand Down Expand Up @@ -740,34 +734,6 @@ internal static unsafe int IndexOfNullByte(byte* searchSpace)
}
}
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
if (offset < (nuint)(uint)Length)
{
lengthToExamine = GetByteVectorSpanLength(offset, Length);

while (lengthToExamine > offset)
{
Vector<byte> matches = Vector.Equals(Vector<byte>.Zero, Vector.Load(searchSpace + offset));
if (Vector<byte>.Zero.Equals(matches))
{
offset += (nuint)Vector<byte>.Count;
continue;
}

// Find offset of first match and add to current offset
return (int)offset + LocateFirstFoundByte(matches);
}

if (offset < (nuint)(uint)Length)
{
lengthToExamine = ((nuint)(uint)Length - offset);
goto SequentialScan;
}
}
}
#endif

ThrowMustBeNullTerminatedString();
Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
Expand Down Expand Up @@ -945,36 +911,6 @@ public static unsafe bool SequenceEqual(ref byte first, ref byte second, nuint l
goto NotEqual;
}
}
#if MONO
else if (Vector.IsHardwareAccelerated && length >= (nuint)Vector<byte>.Count)
{
nuint offset = 0;
nuint lengthToExamine = length - (nuint)Vector<byte>.Count;
// Unsigned, so it shouldn't have overflowed larger than length (rather than negative)
Debug.Assert(lengthToExamine < length);
if (lengthToExamine > 0)
{
do
{
if (LoadVector(ref first, offset) != LoadVector(ref second, offset))
{
goto NotEqual;
}
offset += (nuint)Vector<byte>.Count;
} while (lengthToExamine > offset);
}

// Do final compare as Vector<byte>.Count from end rather than start
if (LoadVector(ref first, lengthToExamine) == LoadVector(ref second, lengthToExamine))
{
// C# compiler inverts this test, making the outer goto the conditional jmp.
goto Equal;
}

// This becomes a conditional jmp forward to not favor it.
goto NotEqual;
}
#endif

#if TARGET_64BIT
if (Vector128.IsHardwareAccelerated)
Expand Down Expand Up @@ -1184,24 +1120,6 @@ public static unsafe int SequenceCompareTo(ref byte first, int firstLength, ref
goto BytewiseCheck;
}
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
if (lengthToExamine > (nuint)Vector<byte>.Count)
{
lengthToExamine -= (nuint)Vector<byte>.Count;
while (lengthToExamine > offset)
{
if (LoadVector(ref first, offset) != LoadVector(ref second, offset))
{
goto BytewiseCheck;
}
offset += (nuint)Vector<byte>.Count;
}
goto BytewiseCheck;
}
}
#endif

if (lengthToExamine > (nuint)sizeof(nuint))
{
Expand Down Expand Up @@ -1323,69 +1241,6 @@ public static nuint CommonPrefixLength(ref byte first, ref byte second, nuint le
return i + uint.TrailingZeroCount(mask);
}

#if MONO
// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundByte(Vector<byte> match)
{
var vector64 = Vector.AsVectorUInt64(match);
ulong candidate = 0;
int i = 0;
// Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
for (; i < Vector<ulong>.Count; i++)
{
candidate = vector64[i];
if (candidate != 0)
{
break;
}
}

// Single LEA instruction with jitted const (using function result)
return i * 8 + LocateFirstFoundByte(candidate);
}

// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateLastFoundByte(Vector<byte> match)
{
var vector64 = Vector.AsVectorUInt64(match);
ulong candidate = 0;
int i = Vector<ulong>.Count - 1;

// This pattern is only unrolled by the Jit if the limit is Vector<T>.Count
// As such, we need a dummy iteration variable for that condition to be satisfied
for (int j = 0; j < Vector<ulong>.Count; j++)
{
candidate = vector64[i];
if (candidate != 0)
{
break;
}

i--;
}

// Single LEA instruction with jitted const (using function result)
return i * 8 + LocateLastFoundByte(candidate);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector<byte> LoadVector(ref byte start, nuint offset)
=> Unsafe.ReadUnaligned<Vector<byte>>(ref Unsafe.AddByteOffset(ref start, offset));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint GetByteVectorSpanLength(nuint offset, int length)
=> (nuint)(uint)((length - (int)offset) & ~(Vector<byte>.Count - 1));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nuint UnalignedCountVector(byte* searchSpace)
{
nint unaligned = (nint)searchSpace & (Vector<byte>.Count - 1);
return (nuint)((Vector<byte>.Count - unaligned) & (Vector<byte>.Count - 1));
}
#endif

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundByte(ulong match)
=> BitOperations.TrailingZeroCount(match) >> 3;
Expand Down
85 changes: 0 additions & 85 deletions src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -546,13 +546,6 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
// Needs to be double length to allow us to align the data first.
lengthToExamine = UnalignedCountVector128(searchSpace);
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
// Needs to be double length to allow us to align the data first.
lengthToExamine = UnalignedCountVector(searchSpace);
}
#endif

SequentialScan:
// In the non-vector case lengthToExamine is the total length.
Expand Down Expand Up @@ -850,42 +843,6 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
}
}
}
#if MONO
else if (Vector.IsHardwareAccelerated)
{
if (offset < length)
{
Debug.Assert(length - offset >= Vector<ushort>.Count);

lengthToExamine = GetCharVectorSpanLength(offset, length);

if (lengthToExamine > 0)
{
do
{
Debug.Assert(lengthToExamine >= Vector<ushort>.Count);

var matches = Vector.Equals(Vector<ushort>.Zero, *(Vector<ushort>*)(searchSpace + (nuint)offset));
if (Vector<ushort>.Zero.Equals(matches))
{
offset += Vector<ushort>.Count;
lengthToExamine -= Vector<ushort>.Count;
continue;
}

// Find offset of first match
return (int)(offset + LocateFirstFoundChar(matches));
} while (lengthToExamine > 0);
}

if (offset < length)
{
lengthToExamine = length - offset;
goto SequentialScan;
}
}
}
#endif

ThrowMustBeNullTerminatedString();
Found3:
Expand All @@ -898,48 +855,6 @@ public static unsafe int IndexOfNullCharacter(char* searchSpace)
return (int)(offset);
}

#if MONO
// Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundChar(Vector<ushort> match)
{
var vector64 = Vector.AsVectorUInt64(match);
ulong candidate = 0;
int i = 0;
// Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
for (; i < Vector<ulong>.Count; i++)
{
candidate = vector64[i];
if (candidate != 0)
{
break;
}
}

// Single LEA instruction with jitted const (using function result)
return i * 4 + LocateFirstFoundChar(candidate);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector<ushort> LoadVector(ref char start, nint offset)
=> Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, offset)));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector<ushort> LoadVector(ref char start, nuint offset)
=> Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, (nint)offset)));

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nint GetCharVectorSpanLength(nint offset, nint length)
=> (length - offset) & ~(Vector<ushort>.Count - 1);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe nint UnalignedCountVector(char* searchSpace)
{
const int ElementsPerByte = sizeof(ushort) / sizeof(byte);
return (nint)(uint)(-(int)searchSpace / ElementsPerByte) & (Vector<ushort>.Count - 1);
}
#endif

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int LocateFirstFoundChar(ulong match)
=> BitOperations.TrailingZeroCount(match) >> 4;
Expand Down