Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace use of target dependent TestZ intrinsic #104488

Merged
merged 6 commits into from
Nov 27, 2024
Merged

Replace use of target dependent TestZ intrinsic #104488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ebe50db
Replace uses of `TestZ` intrinsic
xtqqczze May 12, 2024
a6c7141
Merge branch 'main' into TestZ
tannergooding Oct 16, 2024
4dbcff0
Merge branch 'main' into TestZ
tannergooding Oct 30, 2024
0b7a129
Merge branch 'main' into TestZ
xtqqczze Nov 20, 2024
d21415f
remove pragma warning disable IntrinsicsInSystemPrivateCoreLibConditi…
xtqqczze Nov 23, 2024
653aef8
refactor control flow
xtqqczze Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...raries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64DecoderHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1331,7 +1331,7 @@ public bool TryDecode256Core(
Vector256<sbyte> hi = Avx2.Shuffle(lutHigh, hiNibbles);
Vector256<sbyte> lo = Avx2.Shuffle(lutLow, loNibbles);

if (!Avx.TestZ(lo, hi))
if ((lo & hi) != Vector256<sbyte>.Zero)
{
result = default;
return false;
Expand Down
59 changes: 23 additions & 36 deletions src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1501,10 +1501,10 @@ private static bool VectorContainsNonAsciiChar(Vector128<byte> asciiVector)
{
// max ASCII character is 0b_0111_1111, so the most significant bit (0x80) tells whether it contains non ascii

// prefer architecture specific intrinsic as they offer better perf
// For performance, prefer architecture specific implementation
if (Sse41.IsSupported)
{
return !Sse41.TestZ(asciiVector, Vector128.Create((byte)0x80));
return (asciiVector & Vector128.Create((byte)0x80)) != Vector128<byte>.Zero;
xtqqczze marked this conversation as resolved.
Show resolved Hide resolved
}
else if (AdvSimd.Arm64.IsSupported)
{
Expand All @@ -1520,23 +1520,21 @@ private static bool VectorContainsNonAsciiChar(Vector128<byte> asciiVector)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static bool VectorContainsNonAsciiChar(Vector128<ushort> utf16Vector)
{
// prefer architecture specific intrinsic as they offer better perf
if (Sse2.IsSupported)
// For performance, prefer architecture specific implementation
if (Sse41.IsSupported)
{
if (Sse41.IsSupported)
{
Vector128<ushort> asciiMaskForTestZ = Vector128.Create((ushort)0xFF80);
// If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
return !Sse41.TestZ(utf16Vector.AsInt16(), asciiMaskForTestZ.AsInt16());
}
else
{
Vector128<ushort> asciiMaskForAddSaturate = Vector128.Create((ushort)0x7F80);
// The operation below forces the 0x8000 bit of each WORD to be set iff the WORD element
// has value >= 0x0800 (non-ASCII). Then we'll treat the vector as a BYTE vector in order
// to extract the mask. Reminder: the 0x0080 bit of each WORD should be ignored.
return (Sse2.MoveMask(Sse2.AddSaturate(utf16Vector, asciiMaskForAddSaturate).AsByte()) & 0b_1010_1010_1010_1010) != 0;
}
const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
Vector128<ushort> zeroIsAscii = utf16Vector & Vector128.Create(asciiMask);
// If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
return zeroIsAscii != Vector128<ushort>.Zero;
}
else if (Sse2.IsSupported)
{
Vector128<ushort> asciiMaskForAddSaturate = Vector128.Create((ushort)0x7F80);
// The operation below forces the 0x8000 bit of each WORD to be set iff the WORD element
// has value >= 0x0800 (non-ASCII). Then we'll treat the vector as a BYTE vector in order
// to extract the mask. Reminder: the 0x0080 bit of each WORD should be ignored.
return (Sse2.MoveMask(Sse2.AddSaturate(utf16Vector, asciiMaskForAddSaturate).AsByte()) & 0b_1010_1010_1010_1010) != 0;
}
else if (AdvSimd.Arm64.IsSupported)
{
Expand All @@ -1557,18 +1555,10 @@ internal static bool VectorContainsNonAsciiChar(Vector128<ushort> utf16Vector)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static bool VectorContainsNonAsciiChar(Vector256<ushort> utf16Vector)
{
if (Avx.IsSupported)
{
Vector256<ushort> asciiMaskForTestZ = Vector256.Create((ushort)0xFF80);
return !Avx.TestZ(utf16Vector.AsInt16(), asciiMaskForTestZ.AsInt16());
}
else
{
const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
Vector256<ushort> zeroIsAscii = utf16Vector & Vector256.Create(asciiMask);
// If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
return zeroIsAscii != Vector256<ushort>.Zero;
}
const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
Vector256<ushort> zeroIsAscii = utf16Vector & Vector256.Create(asciiMask);
// If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
return zeroIsAscii != Vector256<ushort>.Zero;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
Expand Down Expand Up @@ -1601,14 +1591,13 @@ private static bool AllCharsInVectorAreAscii<T>(Vector128<T> vector)
if (typeof(T) == typeof(byte))
{
return
Sse41.IsSupported ? Sse41.TestZ(vector.AsByte(), Vector128.Create((byte)0x80)) :
Sse41.IsSupported ? (vector.AsByte() & Vector128.Create((byte)0x80)) == Vector128<byte>.Zero :
xtqqczze marked this conversation as resolved.
Show resolved Hide resolved
AdvSimd.Arm64.IsSupported ? AllBytesInUInt64AreAscii(AdvSimd.Arm64.MaxPairwise(vector.AsByte(), vector.AsByte()).AsUInt64().ToScalar()) :
vector.AsByte().ExtractMostSignificantBits() == 0;
}
else
{
return
Sse41.IsSupported ? Sse41.TestZ(vector.AsUInt16(), Vector128.Create((ushort)0xFF80)) :
AdvSimd.Arm64.IsSupported ? AllCharsInUInt64AreAscii(AdvSimd.Arm64.MaxPairwise(vector.AsUInt16(), vector.AsUInt16()).AsUInt64().ToScalar()) :
(vector.AsUInt16() & Vector128.Create((ushort)0xFF80)) == Vector128<ushort>.Zero;
}
Expand All @@ -1624,14 +1613,12 @@ private static bool AllCharsInVectorAreAscii<T>(Vector256<T> vector)
if (typeof(T) == typeof(byte))
{
return
Avx.IsSupported ? Avx.TestZ(vector.AsByte(), Vector256.Create((byte)0x80)) :
Avx.IsSupported ? (vector.AsByte() & Vector256.Create((byte)0x80)) == Vector256<byte>.Zero:
vector.AsByte().ExtractMostSignificantBits() == 0;
}
else
{
return
Avx.IsSupported ? Avx.TestZ(vector.AsUInt16(), Vector256.Create((ushort)0xFF80)) :
(vector.AsUInt16() & Vector256.Create((ushort)0xFF80)) == Vector256<ushort>.Zero;
return (vector.AsUInt16() & Vector256.Create((ushort)0xFF80)) == Vector256<ushort>.Zero;
}
}

Expand Down
16 changes: 8 additions & 8 deletions src/libraries/System.Private.CoreLib/src/System/Text/Latin1Utility.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ private static unsafe nuint GetIndexOfFirstNonLatin1Char_Sse2(char* pBuffer, nui
{
// If a non-Latin-1 bit is set in any WORD of the combined vector, we have seen non-Latin-1 data.
// Jump to the non-Latin-1 handler to figure out which particular vector contained non-Latin-1 data.
if (!Sse41.TestZ(combinedVector, latin1MaskForTestZ))
if ((combinedVector & latin1MaskForTestZ) != Vector128<ushort>.Zero)
{
goto FoundNonLatin1DataInFirstOrSecondVector;
}
Expand Down Expand Up @@ -312,7 +312,7 @@ private static unsafe nuint GetIndexOfFirstNonLatin1Char_Sse2(char* pBuffer, nui
{
// If a non-Latin-1 bit is set in any WORD of the combined vector, we have seen non-Latin-1 data.
// Jump to the non-Latin-1 handler to figure out which particular vector contained non-Latin-1 data.
if (!Sse41.TestZ(firstVector, latin1MaskForTestZ))
if ((firstVector & latin1MaskForTestZ) != Vector128<ushort>.Zero)
{
goto FoundNonLatin1DataInFirstVector;
}
Expand Down Expand Up @@ -347,7 +347,7 @@ private static unsafe nuint GetIndexOfFirstNonLatin1Char_Sse2(char* pBuffer, nui
{
// If a non-Latin-1 bit is set in any WORD of the combined vector, we have seen non-Latin-1 data.
// Jump to the non-Latin-1 handler to figure out which particular vector contained non-Latin-1 data.
if (!Sse41.TestZ(firstVector, latin1MaskForTestZ))
if ((firstVector & latin1MaskForTestZ) != Vector128<ushort>.Zero)
{
goto FoundNonLatin1DataInFirstVector;
}
Expand Down Expand Up @@ -381,7 +381,7 @@ private static unsafe nuint GetIndexOfFirstNonLatin1Char_Sse2(char* pBuffer, nui
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(firstVector, latin1MaskForTestZ))
if ((firstVector & latin1MaskForTestZ) != Vector128<ushort>.Zero)
{
goto FoundNonLatin1DataInFirstVector;
}
Expand Down Expand Up @@ -795,7 +795,7 @@ private static unsafe nuint NarrowUtf16ToLatin1_Sse2(char* pUtf16Buffer, byte* p
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(utf16VectorFirst, latin1MaskForTestZ))
if ((utf16VectorFirst & latin1MaskForTestZ) != Vector128<short>.Zero)
{
return 0;
}
Expand Down Expand Up @@ -837,7 +837,7 @@ private static unsafe nuint NarrowUtf16ToLatin1_Sse2(char* pUtf16Buffer, byte* p
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(utf16VectorFirst, latin1MaskForTestZ))
if ((utf16VectorFirst & latin1MaskForTestZ) != Vector128<short>.Zero)
{
goto Finish;
}
Expand Down Expand Up @@ -878,7 +878,7 @@ private static unsafe nuint NarrowUtf16ToLatin1_Sse2(char* pUtf16Buffer, byte* p
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(combinedVector, latin1MaskForTestZ))
if ((combinedVector & latin1MaskForTestZ) != Vector128<short>.Zero)
{
goto FoundNonLatin1DataInLoop;
}
Expand Down Expand Up @@ -914,7 +914,7 @@ private static unsafe nuint NarrowUtf16ToLatin1_Sse2(char* pUtf16Buffer, byte* p
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(utf16VectorFirst, latin1MaskForTestZ))
if ((utf16VectorFirst & latin1MaskForTestZ) != Vector128<short>.Zero)
{
goto Finish; // found non-Latin-1 data
}
Expand Down
2 changes: 1 addition & 1 deletion src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -968,7 +968,7 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt
}
else if (Sse41.IsSupported)
{
if (!Sse41.TestZ(utf16Data, nonAsciiUtf16DataMask))
if ((utf16Data & nonAsciiUtf16DataMask) != Vector128<short>.Zero)
{
goto LoopTerminatedDueToNonAsciiDataInVectorLocal;
}
Expand Down
Loading