From 2733ae278e7190cad661734683977278841c94e3 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Fri, 28 May 2021 14:39:46 -0700 Subject: [PATCH 01/18] Add CRC-32 and the basic conformance tests --- .../System.IO.Hashing/Directory.Build.props | 11 + .../System.IO.Hashing/System.IO.Hashing.sln | 104 ++++ .../pkg/System.IO.Hashing.pkgproj | 9 + .../ref/System.IO.Hashing.cs | 40 ++ .../src/Resources/Strings.resx | 126 ++++ .../src/System.IO.Hashing.csproj | 32 + .../src/System/IO/Hashing/Crc32.Table.cs | 40 ++ .../src/System/IO/Hashing/Crc32.cs | 168 +++++ .../Hashing/NonCryptographicHashAlgorithm.cs | 326 ++++++++++ .../System.IO.Hashing/tests/Crc32Tests.cs | 142 +++++ .../tests/NonCryptoHashBaseTests.cs | 581 ++++++++++++++++++ .../tests/NonCryptoHashTestDriver.cs | 361 +++++++++++ .../tests/System.IO.Hashing.Tests.csproj | 17 + 13 files changed, 1957 insertions(+) create mode 100644 src/libraries/System.IO.Hashing/Directory.Build.props create mode 100644 src/libraries/System.IO.Hashing/System.IO.Hashing.sln create mode 100644 src/libraries/System.IO.Hashing/pkg/System.IO.Hashing.pkgproj create mode 100644 src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs create mode 100644 src/libraries/System.IO.Hashing/src/Resources/Strings.resx create mode 100644 src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Table.cs create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/NonCryptographicHashAlgorithm.cs create mode 100644 src/libraries/System.IO.Hashing/tests/Crc32Tests.cs create mode 100644 src/libraries/System.IO.Hashing/tests/NonCryptoHashBaseTests.cs create mode 100644 src/libraries/System.IO.Hashing/tests/NonCryptoHashTestDriver.cs create mode 100644 src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj diff --git a/src/libraries/System.IO.Hashing/Directory.Build.props b/src/libraries/System.IO.Hashing/Directory.Build.props new file mode 100644 index 00000000000000..b7ff7c180adf80 --- /dev/null +++ b/src/libraries/System.IO.Hashing/Directory.Build.props @@ -0,0 +1,11 @@ + + + + Open + Provides non-cryptographic hash algorithms, such as CRC-32. + +Commonly Used Types: +System.IO.Hashing.Crc32 +System.IO.Hashing.Crc64 + + diff --git a/src/libraries/System.IO.Hashing/System.IO.Hashing.sln b/src/libraries/System.IO.Hashing/System.IO.Hashing.sln new file mode 100644 index 00000000000000..974d540529ab4b --- /dev/null +++ b/src/libraries/System.IO.Hashing/System.IO.Hashing.sln @@ -0,0 +1,104 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.31320.298 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestUtilities", "..\Common\tests\TestUtilities\TestUtilities.csproj", "{F94DE827-A426-45CB-AE6E-4E1C154B5386}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Win32.Registry", "..\Microsoft.Win32.Registry\ref\Microsoft.Win32.Registry.csproj", "{38CB62AF-FD52-433E-AB63-CB07BA1D2CD8}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Collections.Immutable", "..\System.Collections.Immutable\src\System.Collections.Immutable.csproj", "{B51B75BA-A1AF-42FB-9845-E9AAC42CFB22}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.IO.Hashing", "ref\System.IO.Hashing.csproj", "{1548AC5C-27FD-4B46-A930-C168D622FAB0}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.IO.Hashing", "src\System.IO.Hashing.csproj", "{A078A4EB-27E8-42B1-BD44-3807732A4560}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.IO.Hashing.Tests", "tests\System.IO.Hashing.Tests.csproj", "{2E6DAC1B-9054-40AF-AF72-4C2DD7BD9294}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Reflection.Metadata", "..\System.Reflection.Metadata\src\System.Reflection.Metadata.csproj", "{0E49FF32-6CC1-42B0-AF30-25098C7DA18F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Runtime.CompilerServices.Unsafe", "..\System.Runtime.CompilerServices.Unsafe\ref\System.Runtime.CompilerServices.Unsafe.csproj", "{696A5E1C-B7B4-4EA0-AE2A-3FDA1C50F4D9}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Runtime.CompilerServices.Unsafe", "..\System.Runtime.CompilerServices.Unsafe\src\System.Runtime.CompilerServices.Unsafe.ilproj", "{E5280823-9A9A-4314-B0C3-B983A7D50C85}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Security.AccessControl", "..\System.Security.AccessControl\ref\System.Security.AccessControl.csproj", "{A7AD2290-A979-4139-9C71-BBCD5594180D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Security.Principal.Windows", "..\System.Security.Principal.Windows\ref\System.Security.Principal.Windows.csproj", "{3A873063-B154-4186-914D-1F4F3236101E}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{3A73868D-BC2F-483A-B51B-0F3C862BBE85}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ref", "ref", "{3F027944-9702-4DA5-A7E2-042D5037ABD3}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{BC246661-9887-4096-BF1A-AC25060547B2}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {F94DE827-A426-45CB-AE6E-4E1C154B5386}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F94DE827-A426-45CB-AE6E-4E1C154B5386}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F94DE827-A426-45CB-AE6E-4E1C154B5386}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F94DE827-A426-45CB-AE6E-4E1C154B5386}.Release|Any CPU.Build.0 = Release|Any CPU + {38CB62AF-FD52-433E-AB63-CB07BA1D2CD8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {38CB62AF-FD52-433E-AB63-CB07BA1D2CD8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {38CB62AF-FD52-433E-AB63-CB07BA1D2CD8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {38CB62AF-FD52-433E-AB63-CB07BA1D2CD8}.Release|Any CPU.Build.0 = Release|Any CPU + {B51B75BA-A1AF-42FB-9845-E9AAC42CFB22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B51B75BA-A1AF-42FB-9845-E9AAC42CFB22}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B51B75BA-A1AF-42FB-9845-E9AAC42CFB22}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B51B75BA-A1AF-42FB-9845-E9AAC42CFB22}.Release|Any CPU.Build.0 = Release|Any CPU + {1548AC5C-27FD-4B46-A930-C168D622FAB0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1548AC5C-27FD-4B46-A930-C168D622FAB0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1548AC5C-27FD-4B46-A930-C168D622FAB0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1548AC5C-27FD-4B46-A930-C168D622FAB0}.Release|Any CPU.Build.0 = Release|Any CPU + {A078A4EB-27E8-42B1-BD44-3807732A4560}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A078A4EB-27E8-42B1-BD44-3807732A4560}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A078A4EB-27E8-42B1-BD44-3807732A4560}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A078A4EB-27E8-42B1-BD44-3807732A4560}.Release|Any CPU.Build.0 = Release|Any CPU + {2E6DAC1B-9054-40AF-AF72-4C2DD7BD9294}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2E6DAC1B-9054-40AF-AF72-4C2DD7BD9294}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2E6DAC1B-9054-40AF-AF72-4C2DD7BD9294}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2E6DAC1B-9054-40AF-AF72-4C2DD7BD9294}.Release|Any CPU.Build.0 = Release|Any CPU + {0E49FF32-6CC1-42B0-AF30-25098C7DA18F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0E49FF32-6CC1-42B0-AF30-25098C7DA18F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0E49FF32-6CC1-42B0-AF30-25098C7DA18F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0E49FF32-6CC1-42B0-AF30-25098C7DA18F}.Release|Any CPU.Build.0 = Release|Any CPU + {696A5E1C-B7B4-4EA0-AE2A-3FDA1C50F4D9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {696A5E1C-B7B4-4EA0-AE2A-3FDA1C50F4D9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {696A5E1C-B7B4-4EA0-AE2A-3FDA1C50F4D9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {696A5E1C-B7B4-4EA0-AE2A-3FDA1C50F4D9}.Release|Any CPU.Build.0 = Release|Any CPU + {E5280823-9A9A-4314-B0C3-B983A7D50C85}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5280823-9A9A-4314-B0C3-B983A7D50C85}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5280823-9A9A-4314-B0C3-B983A7D50C85}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5280823-9A9A-4314-B0C3-B983A7D50C85}.Release|Any CPU.Build.0 = Release|Any CPU + {A7AD2290-A979-4139-9C71-BBCD5594180D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A7AD2290-A979-4139-9C71-BBCD5594180D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A7AD2290-A979-4139-9C71-BBCD5594180D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A7AD2290-A979-4139-9C71-BBCD5594180D}.Release|Any CPU.Build.0 = Release|Any CPU + {3A873063-B154-4186-914D-1F4F3236101E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3A873063-B154-4186-914D-1F4F3236101E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3A873063-B154-4186-914D-1F4F3236101E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3A873063-B154-4186-914D-1F4F3236101E}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {F94DE827-A426-45CB-AE6E-4E1C154B5386} = {3A73868D-BC2F-483A-B51B-0F3C862BBE85} + {38CB62AF-FD52-433E-AB63-CB07BA1D2CD8} = {3F027944-9702-4DA5-A7E2-042D5037ABD3} + {B51B75BA-A1AF-42FB-9845-E9AAC42CFB22} = {BC246661-9887-4096-BF1A-AC25060547B2} + {1548AC5C-27FD-4B46-A930-C168D622FAB0} = {3F027944-9702-4DA5-A7E2-042D5037ABD3} + {A078A4EB-27E8-42B1-BD44-3807732A4560} = {BC246661-9887-4096-BF1A-AC25060547B2} + {2E6DAC1B-9054-40AF-AF72-4C2DD7BD9294} = {3A73868D-BC2F-483A-B51B-0F3C862BBE85} + {0E49FF32-6CC1-42B0-AF30-25098C7DA18F} = {BC246661-9887-4096-BF1A-AC25060547B2} + {696A5E1C-B7B4-4EA0-AE2A-3FDA1C50F4D9} = {3F027944-9702-4DA5-A7E2-042D5037ABD3} + {E5280823-9A9A-4314-B0C3-B983A7D50C85} = {BC246661-9887-4096-BF1A-AC25060547B2} + {A7AD2290-A979-4139-9C71-BBCD5594180D} = {3F027944-9702-4DA5-A7E2-042D5037ABD3} + {3A873063-B154-4186-914D-1F4F3236101E} = {3F027944-9702-4DA5-A7E2-042D5037ABD3} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {4DF081FC-DC0D-4317-AB2C-4294B9FE6257} + EndGlobalSection +EndGlobal diff --git a/src/libraries/System.IO.Hashing/pkg/System.IO.Hashing.pkgproj b/src/libraries/System.IO.Hashing/pkg/System.IO.Hashing.pkgproj new file mode 100644 index 00000000000000..0687eec5e0accf --- /dev/null +++ b/src/libraries/System.IO.Hashing/pkg/System.IO.Hashing.pkgproj @@ -0,0 +1,9 @@ + + + + + net461;netcoreapp2.0;uap10.0.16299;$(AllXamarinFrameworks) + + + + diff --git a/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs new file mode 100644 index 00000000000000..eedadf5ad1725b --- /dev/null +++ b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// ------------------------------------------------------------------------------ +// Changes to this file must follow the https://aka.ms/api-review process. +// ------------------------------------------------------------------------------ + +namespace System.IO.Hashing +{ + public sealed partial class Crc32 : System.IO.Hashing.NonCryptographicHashAlgorithm + { + public Crc32() : base (default(int)) { } + public override void Append(System.ReadOnlySpan source) { } + protected override void GetCurrentHashCore(System.Span destination) { } + protected override void GetHashAndResetCore(System.Span destination) { } + public static byte[] Hash(byte[] source) { throw null; } + public static byte[] Hash(System.ReadOnlySpan source) { throw null; } + public static int Hash(System.ReadOnlySpan source, System.Span destination) { throw null; } + public override void Reset() { } + public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } + } + public abstract partial class NonCryptographicHashAlgorithm + { + protected NonCryptographicHashAlgorithm(int hashLengthInBytes) { } + public int HashLengthInBytes { get { throw null; } } + public void Append(byte[] source) { } + public void Append(System.IO.Stream stream) { } + public abstract void Append(System.ReadOnlySpan source); + public System.Threading.Tasks.Task AppendAsync(System.IO.Stream stream, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public byte[] GetCurrentHash() { throw null; } + public int GetCurrentHash(System.Span destination) { throw null; } + protected abstract void GetCurrentHashCore(System.Span destination); + public byte[] GetHashAndReset() { throw null; } + public int GetHashAndReset(System.Span destination) { throw null; } + protected virtual void GetHashAndResetCore(System.Span destination) { } + public override int GetHashCode() { throw null; } + public abstract void Reset(); + public bool TryGetCurrentHash(System.Span destination, out int bytesWritten) { throw null; } + public bool TryGetHashAndReset(System.Span destination, out int bytesWritten) { throw null; } + } +} diff --git a/src/libraries/System.IO.Hashing/src/Resources/Strings.resx b/src/libraries/System.IO.Hashing/src/Resources/Strings.resx new file mode 100644 index 00000000000000..a8101c05698318 --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/Resources/Strings.resx @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Destination is too short. + + + The GetHashCode method is not supported on this object. Use GetCurrentHash or GetHashAndReset to retrieve the hash code computed by this object. + + \ No newline at end of file diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj new file mode 100644 index 00000000000000..4766f6d027d8ad --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -0,0 +1,32 @@ + + + true + enable + + netstandard2.0;net461 + true + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Table.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Table.cs new file mode 100644 index 00000000000000..b8188f1f0af59b --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Table.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.IO.Hashing +{ + public sealed partial class Crc32 : NonCryptographicHashAlgorithm + { + // Pre-computed CRC-32 transition table. + // While this implementation is based on the standard CRC-32 polynomial, + // x32 + x26 + x23 + x22 + x16 + x12 + x11 + x10 + x8 + x7 + x5 + x4 + x2 + x1 + x0, + // this version uses reflected bit ordering, so 0x04C11DB7 becomes 0xEDB88320 + private static readonly uint[] s_crcLookup = GenerateReflectedTable(0xEDB88320u); + + private static uint[] GenerateReflectedTable(uint reflectedPolynomial) + { + uint[] table = new uint[256]; + + for (int i = 0; i < 256; i++) + { + uint val = unchecked((uint)i); + + for (int j = 0; j < 8; j++) + { + if ((val & 0b0000_0001) == 0) + { + val >>= 1; + } + else + { + val = (val >> 1) ^ reflectedPolynomial; + } + } + + table[i] = val; + } + + return table; + } + } +} diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs new file mode 100644 index 00000000000000..4f904ebaa64b07 --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.cs @@ -0,0 +1,168 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers.Binary; + +namespace System.IO.Hashing +{ + /// + /// Provides an implementation of the CRC-32 algorithm, as used in + /// ITU-T V.42 and IEEE 802.3. + /// + /// + /// + /// This implementation emits the answer in the Little Endian byte order so that + /// the CRC residue relationship (CRC(message concat CRC(message))) is a fixed value) holds. + /// For CRC-32 this stable output is the byte sequence { 0x1C, 0xDF, 0x44, 0x21 }, + /// the Little Endian representation of 0x2144DF1C. + /// + /// + /// There are multiple, incompatible, definitions of a 32-bit cyclic redundancy + /// check (CRC) algorithm. When interoperating with another system, ensure that you + /// are using the same definition. The definition used by this implementation is not + /// compatible with the cyclic redundancy check described in ITU-T I.363.5. + /// + /// + public sealed partial class Crc32 : NonCryptographicHashAlgorithm + { + private const uint InitialState = 0xFFFF_FFFFu; + private const int Size = sizeof(uint); + + private uint _crc = InitialState; + + /// + /// Initializes a new instance of the class. + /// + public Crc32() + : base(Size) + { + } + + /// + /// Appends the contents of to the data already + /// processed for the current hash computation. + /// + /// The data to process. + public override void Append(ReadOnlySpan source) + { + _crc = Update(_crc, source); + } + + /// + /// Resets the hash computation to the initial state. + /// + public override void Reset() + { + _crc = InitialState; + } + + /// + /// Writes the computed hash value to + /// without modifying accumulated state. + /// + /// The buffer that receives the computed hash value. + protected override void GetCurrentHashCore(Span destination) + { + // The finalization step of the CRC is to perform the ones' complement. + BinaryPrimitives.WriteUInt32LittleEndian(destination, ~_crc); + } + + /// + /// Writes the computed hash value to + /// then clears the accumulated state. + /// + protected override void GetHashAndResetCore(Span destination) + { + BinaryPrimitives.WriteUInt32LittleEndian(destination, ~_crc); + _crc = InitialState; + } + + /// + /// Computes the CRC-32 hash of the provided data. + /// + /// The data to hash. + /// The CRC-32 hash of the provided data. + /// + /// is . + /// + public static byte[] Hash(byte[] source) + { + if (source is null) + throw new ArgumentNullException(nameof(source)); + + return Hash(new ReadOnlySpan(source)); + } + + /// + /// Computes the CRC-32 hash of the provided data. + /// + /// The data to hash. + /// The CRC-32 hash of the provided data. + public static byte[] Hash(ReadOnlySpan source) + { + byte[] ret = new byte[Size]; + StaticHash(source, ret); + return ret; + } + + /// + /// Attempts to compute the CRC-32 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// On success, receives the number of bytes written to . + /// + /// + /// if is long enough to receive + /// the computed hash value (4 bytes); otherwise, . + /// + public static bool TryHash(ReadOnlySpan source, Span destination, out int bytesWritten) + { + if (destination.Length < Size) + { + bytesWritten = 0; + return false; + } + + bytesWritten = StaticHash(source, destination); + return true; + } + + /// + /// Computes the CRC-32 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// The number of bytes written to . + /// + public static int Hash(ReadOnlySpan source, Span destination) + { + if (destination.Length < Size) + throw new ArgumentException(SR.Argument_DestinationTooShort, nameof(destination)); + + return StaticHash(source, destination); + } + + private static int StaticHash(ReadOnlySpan source, Span destination) + { + uint crc = InitialState; + crc = Update(crc, source); + BinaryPrimitives.WriteUInt32LittleEndian(destination, ~crc); + return Size; + } + + private static uint Update(uint crc, ReadOnlySpan source) + { + for (int i = 0; i < source.Length; i++) + { + byte idx = (byte)crc; + idx ^= source[i]; + crc = s_crcLookup[idx] ^ (crc >> 8); + } + + return crc; + } + } +} diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/NonCryptographicHashAlgorithm.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/NonCryptographicHashAlgorithm.cs new file mode 100644 index 00000000000000..ba9e664cb9bd92 --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/NonCryptographicHashAlgorithm.cs @@ -0,0 +1,326 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.ComponentModel; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace System.IO.Hashing +{ + /// + /// Represents a non-cryptographic hash algorithm. + /// + public abstract class NonCryptographicHashAlgorithm + { + /// + /// Gets the number of bytes produced from this hash algorithm. + /// + /// The number of bytes produced from this hash algorithm. + public int HashLengthInBytes { get; } + + /// + /// Called from constructors in derived classes to initialize the + /// class. + /// + /// + /// The number of bytes produced from this hash algorithm. + /// + /// + /// is less than 1. + /// + protected NonCryptographicHashAlgorithm(int hashLengthInBytes) + { + if (hashLengthInBytes < 1) + throw new ArgumentOutOfRangeException(nameof(hashLengthInBytes)); + + HashLengthInBytes = hashLengthInBytes; + } + + /// + /// When overridden in a derived class, + /// appends the contents of to the data already + /// processed for the current hash computation. + /// + /// The data to process. + public abstract void Append(ReadOnlySpan source); + + /// + /// When overridden in a derived class, + /// resets the hash computation to the initial state. + /// + public abstract void Reset(); + + /// + /// When overridden in a derived class, + /// writes the computed hash value to + /// without modifying accumulated state. + /// + /// The buffer that receives the computed hash value. + /// + /// + /// Implementations of this method must write exactly + /// bytes to . + /// Do not assume that the buffer was zero-initialized. + /// + /// + /// The class validates the + /// size of the buffer before calling this method, and slices the span + /// down to be exactly in length. + /// + /// + protected abstract void GetCurrentHashCore(Span destination); + + /// + /// Appends the contents of to the data already + /// processed for the current hash computation. + /// + /// The data to process. + /// + /// is . + /// + public void Append(byte[] source) + { + if (source is null) + throw new ArgumentNullException(nameof(source)); + + Append(new ReadOnlySpan(source)); + } + + /// + /// Appends the contents of to the data already + /// processed for the current hash computation. + /// + /// The data to process. + /// + /// is . + /// + /// + public void Append(Stream stream) + { + if (stream is null) + throw new ArgumentNullException(nameof(stream)); + + byte[] buffer = ArrayPool.Shared.Rent(4096); + + while (true) + { + int read = stream.Read(buffer, 0, buffer.Length); + + if (read == 0) + { + break; + } + + Append(new ReadOnlySpan(buffer, 0, read)); + } + + ArrayPool.Shared.Return(buffer); + } + + /// + /// Asychronously reads the contents of + /// and appends them to the data already + /// processed for the current hash computation. + /// + /// The data to process. + /// + /// The token to monitor for cancellation requests. + /// The default value is . + /// + /// + /// is . + /// + public Task AppendAsync(Stream stream, CancellationToken cancellationToken = default) + { + if (stream is null) + throw new ArgumentNullException(nameof(stream)); + + return AppendAsyncCore(stream, cancellationToken); + } + + private async Task AppendAsyncCore(Stream stream, CancellationToken cancellationToken) + { + byte[] buffer = ArrayPool.Shared.Rent(4096); + + while (true) + { + int read = await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false); + + if (read == 0) + { + break; + } + + Append(new ReadOnlySpan(buffer, 0, read)); + } + + ArrayPool.Shared.Return(buffer); + } + + /// + /// Gets the current computed hash value without modifying accumulated state. + /// + public byte[] GetCurrentHash() + { + byte[] ret = new byte[HashLengthInBytes]; + GetCurrentHashCore(ret); + return ret; + } + + /// + /// Attempts to write the computed hash value to + /// without modifying accumulated state. + /// + /// The buffer that receives the computed hash value. + /// + /// On success, receives the number of bytes written to . + /// + /// + /// if is long enough to receive + /// the computed hash value; otherwise, . + /// + public bool TryGetCurrentHash(Span destination, out int bytesWritten) + { + if (destination.Length < HashLengthInBytes) + { + bytesWritten = 0; + return false; + } + + GetCurrentHashCore(destination.Slice(0, HashLengthInBytes)); + bytesWritten = HashLengthInBytes; + return true; + } + + /// + /// Writes the computed hash value to + /// without modifying accumulated state. + /// + /// The buffer that receives the computed hash value. + /// + /// The number of bytes written to , + /// which is always . + /// + /// + /// is shorter than . + /// + public int GetCurrentHash(Span destination) + { + if (destination.Length < HashLengthInBytes) + { + throw new ArgumentException(SR.Argument_DestinationTooShort, nameof(destination)); + } + + GetCurrentHashCore(destination.Slice(0, HashLengthInBytes)); + return HashLengthInBytes; + } + + /// + /// Gets the current computed hash value and clears the accumulated state. + /// + public byte[] GetHashAndReset() + { + byte[] ret = new byte[HashLengthInBytes]; + GetHashAndResetCore(ret); + return ret; + } + + /// + /// Attempts to write the computed hash value to . + /// If successful, clears the accumulated state. + /// + /// The buffer that receives the computed hash value. + /// + /// On success, receives the number of bytes written to . + /// + /// + /// and clears the accumulated state + /// if is long enough to receive + /// the computed hash value; otherwise, . + /// + public bool TryGetHashAndReset(Span destination, out int bytesWritten) + { + if (destination.Length < HashLengthInBytes) + { + bytesWritten = 0; + return false; + } + + GetHashAndResetCore(destination.Slice(0, HashLengthInBytes)); + bytesWritten = HashLengthInBytes; + return true; + } + + /// + /// Writes the computed hash value to + /// then clears the accumulated state. + /// + /// The buffer that receives the computed hash value. + /// + /// The number of bytes written to , + /// which is always . + /// + /// + /// is shorter than . + /// + public int GetHashAndReset(Span destination) + { + if (destination.Length < HashLengthInBytes) + { + throw new ArgumentException(SR.Argument_DestinationTooShort, nameof(destination)); + } + + GetHashAndResetCore(destination.Slice(0, HashLengthInBytes)); + return HashLengthInBytes; + } + + /// + /// Writes the computed hash value to + /// then clears the accumulated state. + /// + /// The buffer that receives the computed hash value. + /// + /// + /// Implementations of this method must write exactly + /// bytes to . + /// Do not assume that the buffer was zero-initialized. + /// + /// + /// The class validates the + /// size of the buffer before calling this method, and slices the span + /// down to be exactly in length. + /// + /// + /// The default implementation of this method calls + /// followed by . + /// Overrides of this method do not need to call either of those methods, + /// but must ensure that the caller cannot observe a difference in behavior. + /// + /// + protected virtual void GetHashAndResetCore(Span destination) + { + Debug.Assert(destination.Length == HashLengthInBytes); + + GetCurrentHashCore(destination); + Reset(); + } + + /// + /// This method is not supported and should not be called. + /// Call or + /// instead. + /// + /// This method will always throw a . + /// In all cases. + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("Use GetCurrentHash() to retrieve the computed hash code.", true)] +#pragma warning disable CS0809 // Obsolete member overrides non-obsolete member + public override int GetHashCode() +#pragma warning restore CS0809 // Obsolete member overrides non-obsolete member + { + throw new NotSupportedException(SR.NotSupported_GetHashCode); + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs b/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs new file mode 100644 index 00000000000000..c8f5486c89b17b --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs @@ -0,0 +1,142 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public class Crc32Tests : NonCryptoHashTestDriver + { + public Crc32Tests() : base(new byte[4]) + { + } + + public static IEnumerable TestCases + { + get + { + object[] arr = new object[1]; + + foreach (TestCase testCase in TestCaseDefinitions) + { + arr[0] = testCase; + yield return arr; + } + } + } + + protected static IEnumerable TestCaseDefinitions { get; } = + new[] + { + new TestCase( + "Empty", + "", + "00000000"), + new TestCase( + "One", + "01", + "1BDF05A5"), + new TestCase( + "Zero-Residue", + "00000000", + "1CDF4421"), + new TestCase( + "Zero-InverseResidue", + "FFFFFFFF", + "FFFFFFFF"), + new TestCase( + "Self-test 123456789", + Encoding.ASCII.GetBytes("123456789"), + "2639F4CB"), + new TestCase( + "Self-test residue", + "3132333435363738392639F4CB", + "1CDF4421"), + new TestCase( + "Self-test inverse residue", + "313233343536373839D9C60B34", + "FFFFFFFF"), + new TestCase( + "The quick brown fox jumps over the lazy dog", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog"), + "39A34F41"), + }; + + protected override NonCryptographicHashAlgorithm CreateInstance() => new Crc32(); + + protected override byte[] StaticOneShot(byte[] source) => Crc32.Hash(source); + + protected override byte[] StaticOneShot(ReadOnlySpan source) => Crc32.Hash(source); + + protected override int StaticOneShot(ReadOnlySpan source, Span destination) => + Crc32.Hash(source, destination); + + protected override bool TryStaticOneShot(ReadOnlySpan source, Span destination, out int bytesWritten) => + Crc32.TryHash(source, destination, out bytesWritten); + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocate(TestCase testCase) + { + InstanceAppendAllocateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocateAndReset(TestCase testCase) + { + InstanceAppendAllocateAndResetDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceMultiAppendGetCurrentHash(TestCase testCase) + { + InstanceMultiAppendGetCurrentHashDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyEmptyState(TestCase testCase) + { + InstanceVerifyEmptyStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyResetState(TestCase testCase) + { + InstanceVerifyResetStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotArray(TestCase testCase) + { + StaticVerifyOneShotArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToArray(TestCase testCase) + { + StaticVerifyOneShotSpanToArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToSpan(TestCase testCase) + { + StaticVerifyOneShotSpanToSpanDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyTryOneShot(TestCase testCase) + { + StaticVerifyTryOneShotDriver(testCase); + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/NonCryptoHashBaseTests.cs b/src/libraries/System.IO.Hashing/tests/NonCryptoHashBaseTests.cs new file mode 100644 index 00000000000000..bf72d4ef8513f0 --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/NonCryptoHashBaseTests.cs @@ -0,0 +1,581 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers.Binary; +using System.Threading; +using System.Threading.Tasks; +using Test.IO.Streams; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public static class NonCryptoHashBaseTests + { + [Fact] + public static void ZeroLengthHashIsInvalid() + { + AssertExtensions.Throws( + "hashLengthInBytes", + () => new FlexibleAlgorithm(0)); + } + + [Fact] + public static void NegativeHashLengthIsInvalid() + { + AssertExtensions.Throws( + "hashLengthInBytes", + () => new FlexibleAlgorithm(-1)); + } + + [Fact] + public static void TryGetCurrentHash_TooSmall() + { + Span buf = stackalloc byte[7]; + FlexibleAlgorithm hash = new FlexibleAlgorithm(buf.Length + 1); + + Assert.True(hash.IsReset); + hash.Append(buf); + Assert.False(hash.IsReset); + + while (true) + { + Assert.False(hash.TryGetCurrentHash(buf, out int written)); + Assert.Equal(0, written); + + if (buf.IsEmpty) + { + break; + } + + buf = buf.Slice(1); + } + + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.False(hash.IsReset); + } + + [Fact] + public static void TryGetCurrentHash_TooBig_Succeeds() + { + Span buf = stackalloc byte[16]; + FlexibleAlgorithm hash = new FlexibleAlgorithm(buf.Length / 2); + int i = 0; + + hash.Append(buf); + + while (buf.Length > hash.HashLengthInBytes) + { + Assert.Equal(i, hash.GetCurrentHashCoreCallCount); + + // TryGetCurrentHash in turn asserts that buf got Sliced down to HashLengthInBytes. + Assert.True(hash.TryGetCurrentHash(buf, out int written)); + Assert.Equal(hash.HashLengthInBytes, written); + + buf = buf.Slice(1); + i++; + } + + Assert.Equal(i, hash.GetCurrentHashCoreCallCount); + Assert.False(hash.IsReset); + } + + [Fact] + public static void GetCurrentHash_TooSmall() + { + byte[] buf = new byte[7]; + FlexibleAlgorithm hash = new FlexibleAlgorithm(buf.Length + 1); + + Assert.True(hash.IsReset); + hash.Append(buf); + Assert.False(hash.IsReset); + + for (int i = 0; i <= buf.Length; i++) + { + AssertExtensions.Throws( + "destination", + () => hash.GetCurrentHash(buf.AsSpan(i))); + } + + Assert.False(hash.IsReset); + } + + [Fact] + public static void GetCurrentHash_TooBig_Succeeds() + { + Span buf = stackalloc byte[16]; + FlexibleAlgorithm hash = new FlexibleAlgorithm(buf.Length / 2); + int i = 0; + + hash.Append(buf); + + while (buf.Length > hash.HashLengthInBytes) + { + Assert.Equal(i, hash.GetCurrentHashCoreCallCount); + + // GetCurrentHash in turn asserts that buf got Sliced down to HashLengthInBytes. + int written = hash.GetCurrentHash(buf); + Assert.Equal(hash.HashLengthInBytes, written); + + buf = buf.Slice(1); + i++; + } + + Assert.Equal(i, hash.GetCurrentHashCoreCallCount); + Assert.False(hash.IsReset); + } + + [Fact] + public static void AllocatingGetCurrentHash_CorrectSize() + { + FlexibleAlgorithm hash = new FlexibleAlgorithm(12); + byte[] ret = hash.GetCurrentHash(); + Assert.Equal(hash.HashLengthInBytes, ret.Length); + Assert.Equal(1, hash.GetCurrentHashCoreCallCount); + } + + [Fact] + public static void DefaultGetHashAndResetDoesWhatItSays() + { + Span buf = stackalloc byte[16]; + FlexibleAlgorithm hash = new FlexibleAlgorithm(buf.Length); + Assert.True(hash.IsReset); + hash.Append(buf); + Assert.False(hash.IsReset); + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + + byte[] ret = hash.GetHashAndReset(); + Assert.True(hash.IsReset); + Assert.Equal(1, hash.GetCurrentHashCoreCallCount); + Assert.Equal(hash.HashLengthInBytes, ret.Length); + + hash.Append(ret); + Assert.False(hash.IsReset); + + Assert.True(hash.TryGetHashAndReset(buf, out int written)); + Assert.True(hash.IsReset); + Assert.Equal(2, hash.GetCurrentHashCoreCallCount); + Assert.Equal(hash.HashLengthInBytes, written); + + hash.Append(ret); + Assert.False(hash.IsReset); + + written = hash.GetHashAndReset(buf); + Assert.True(hash.IsReset); + Assert.Equal(3, hash.GetCurrentHashCoreCallCount); + Assert.Equal(hash.HashLengthInBytes, written); + } + + [Fact] + public static void TryGetHashAndReset_TooSmall() + { + Span buf = stackalloc byte[7]; + FlexibleAlgorithm hash = new FlexibleAlgorithm(buf.Length + 1); + + Assert.True(hash.IsReset); + hash.Append(buf); + Assert.False(hash.IsReset); + + while (true) + { + Assert.False(hash.TryGetHashAndReset(buf, out int written)); + Assert.Equal(0, written); + + if (buf.IsEmpty) + { + break; + } + + buf = buf.Slice(1); + } + + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.False(hash.IsReset); + } + + [Fact] + public static void TryGetHashAndReset_TooBig_Succeeds() + { + Span buf = stackalloc byte[16]; + FlexibleAlgorithm hash = new FlexibleAlgorithm(buf.Length / 2); + int i = 0; + + while (buf.Length > hash.HashLengthInBytes) + { + Assert.Equal(i, hash.GetCurrentHashCoreCallCount); + + hash.Append(buf); + Assert.False(hash.IsReset); + + // TryGetCurrentHash in turn asserts that buf got Sliced down to HashLengthInBytes. + Assert.True(hash.TryGetHashAndReset(buf, out int written)); + Assert.Equal(hash.HashLengthInBytes, written); + Assert.True(hash.IsReset); + + buf = buf.Slice(1); + i++; + } + + Assert.Equal(i, hash.GetCurrentHashCoreCallCount); + Assert.True(hash.IsReset); + } + + [Fact] + public static void GetHashAndReset_TooSmall() + { + byte[] buf = new byte[7]; + FlexibleAlgorithm hash = new FlexibleAlgorithm(buf.Length + 1); + + Assert.True(hash.IsReset); + hash.Append(buf); + Assert.False(hash.IsReset); + + for (int i = 0; i <= buf.Length; i++) + { + AssertExtensions.Throws( + "destination", + () => hash.GetHashAndReset(buf.AsSpan(i))); + } + + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.False(hash.IsReset); + } + + [Fact] + public static void GetHashAndReset_TooBig_Succeeds() + { + Span buf = stackalloc byte[16]; + FlexibleAlgorithm hash = new FlexibleAlgorithm(buf.Length / 2); + int i = 0; + + while (buf.Length > hash.HashLengthInBytes) + { + Assert.Equal(i, hash.GetCurrentHashCoreCallCount); + + hash.Append(buf); + Assert.False(hash.IsReset); + + Assert.True(hash.TryGetHashAndReset(buf, out int written)); + Assert.Equal(hash.HashLengthInBytes, written); + Assert.True(hash.IsReset); + + buf = buf.Slice(1); + i++; + } + + Assert.Equal(i, hash.GetCurrentHashCoreCallCount); + Assert.True(hash.IsReset); + } + + [Fact] + public static void AllocatingGetHashAndReset_CorrectSize() + { + FlexibleAlgorithm hash = new FlexibleAlgorithm(12); + byte[] ret = hash.GetHashAndReset(); + Assert.Equal(hash.HashLengthInBytes, ret.Length); + Assert.Equal(1, hash.GetCurrentHashCoreCallCount); + } + + [Fact] + public static void OverriddenTryGetHashAndReset_TooSmall() + { + Span buf = stackalloc byte[7]; + var hash = new FlexibleAlgorithmOverride(buf.Length + 1); + + Assert.True(hash.IsReset); + hash.Append(buf); + Assert.False(hash.IsReset); + + while (true) + { + Assert.False(hash.TryGetHashAndReset(buf, out int written)); + Assert.Equal(0, written); + + if (buf.IsEmpty) + { + break; + } + + buf = buf.Slice(1); + } + + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.Equal(0, hash.GetHashAndResetCoreCallCount); + Assert.False(hash.IsReset); + } + + [Fact] + public static void OverriddenTryGetHashAndReset_TooBig_Succeeds() + { + Span buf = stackalloc byte[16]; + var hash = new FlexibleAlgorithmOverride(buf.Length / 2); + int i = 0; + + while (buf.Length > hash.HashLengthInBytes) + { + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.Equal(i, hash.GetHashAndResetCoreCallCount); + + hash.Append(buf); + Assert.False(hash.IsReset); + + // TryGetCurrentHash in turn asserts that buf got Sliced down to HashLengthInBytes. + Assert.True(hash.TryGetHashAndReset(buf, out int written)); + Assert.Equal(hash.HashLengthInBytes, written); + Assert.True(hash.IsReset); + + buf = buf.Slice(1); + i++; + } + + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.Equal(i, hash.GetHashAndResetCoreCallCount); + Assert.True(hash.IsReset); + } + + [Fact] + public static void OverriddenGetHashAndReset_TooSmall() + { + byte[] buf = new byte[7]; + var hash = new FlexibleAlgorithmOverride(buf.Length + 1); + + Assert.True(hash.IsReset); + hash.Append(buf); + Assert.False(hash.IsReset); + + for (int i = 0; i <= buf.Length; i++) + { + AssertExtensions.Throws( + "destination", + () => hash.GetHashAndReset(buf.AsSpan(i))); + } + + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.Equal(0, hash.GetHashAndResetCoreCallCount); + Assert.False(hash.IsReset); + } + + [Fact] + public static void OverriddenGetHashAndReset_TooBig_Succeeds() + { + Span buf = stackalloc byte[16]; + var hash = new FlexibleAlgorithmOverride(buf.Length / 2); + int i = 0; + + while (buf.Length > hash.HashLengthInBytes) + { + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.Equal(i, hash.GetHashAndResetCoreCallCount); + + hash.Append(buf); + Assert.False(hash.IsReset); + + Assert.True(hash.TryGetHashAndReset(buf, out int written)); + Assert.Equal(hash.HashLengthInBytes, written); + Assert.True(hash.IsReset); + + buf = buf.Slice(1); + i++; + } + + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.Equal(i, hash.GetHashAndResetCoreCallCount); + Assert.True(hash.IsReset); + } + + [Fact] + public static void OverriddenAllocatingGetHashAndReset_CorrectSize() + { + var hash = new FlexibleAlgorithmOverride(12); + byte[] ret = hash.GetHashAndReset(); + Assert.Equal(hash.HashLengthInBytes, ret.Length); + Assert.Equal(0, hash.GetCurrentHashCoreCallCount); + Assert.Equal(1, hash.GetHashAndResetCoreCallCount); + } + + [Fact] + public static void AppendNullArrayThrows() + { + NonCryptographicHashAlgorithm hash = new FlexibleAlgorithm(5); + AssertExtensions.Throws("source", () => hash.Append((byte[])null)); + } + + [Fact] + public static void AppendNullStreamThrows() + { + NonCryptographicHashAlgorithm hash = new FlexibleAlgorithm(5); + AssertExtensions.Throws("stream", () => hash.Append((Stream)null)); + } + + [Fact] + public static void AppendAsyncNullStreamThrows_OutsideTask() + { + NonCryptographicHashAlgorithm hash = new FlexibleAlgorithm(5); + AssertExtensions.Throws("stream", () => hash.AppendAsync(null)); + } + + [Theory] + [InlineData(0)] + [InlineData(511)] + [InlineData(4097)] + [InlineData(1048581)] + public static void AppendStreamSanityTest(int streamSize) + { + NonCryptographicHashAlgorithm hash = new CountingAlgorithm(); + + using (PositionValueStream stream = new PositionValueStream(streamSize)) + { + hash.Append(stream); + } + + Span val = stackalloc byte[sizeof(int)]; + hash.GetHashAndReset(val); + int res = BinaryPrimitives.ReadInt32LittleEndian(val); + Assert.Equal(streamSize, res); + } + + [Theory] + [InlineData(0)] + [InlineData(511)] + [InlineData(4097)] + [InlineData(1048581)] + public static async Task AppendStreamAsyncSanityTest(int streamSize) + { + NonCryptographicHashAlgorithm hash = new CountingAlgorithm(); + + using (PositionValueStream stream = new PositionValueStream(streamSize)) + { + await hash.AppendAsync(stream); + } + + byte[] val = new byte[sizeof(int)]; + hash.GetHashAndReset(val); + int res = BinaryPrimitives.ReadInt32LittleEndian(val); + Assert.Equal(streamSize, res); + } + + [Fact] + public static void AppendStreamAsyncSupportsCancellation() + { + NonCryptographicHashAlgorithm hash = new CountingAlgorithm(); + + using (PositionValueStream stream = new PositionValueStream(21)) + using (CancellationTokenSource cts = new CancellationTokenSource()) + { + cts.Cancel(); + + Task task = hash.AppendAsync(stream, cts.Token); + Assert.True(task.IsCompleted); + Assert.True(task.IsCanceled); + } + + byte[] val = new byte[sizeof(int)]; + hash.GetHashAndReset(val); + int res = BinaryPrimitives.ReadInt32LittleEndian(val); + Assert.Equal(0, res); + } + + [Fact] + public static void GetHashCode_NotSupported() + { + NonCryptographicHashAlgorithm hash = new CountingAlgorithm(); + Assert.Throws(() => hash.GetHashCode()); + } + private sealed class CountingAlgorithm : NonCryptographicHashAlgorithm + { + private int _count; + + public CountingAlgorithm() + : base(sizeof(int)) + { + } + + public override void Append(ReadOnlySpan source) + { + _count += source.Length; + } + + public override void Reset() + { + _count = 0; + } + + protected override void GetCurrentHashCore(Span destination) + { + BinaryPrimitives.WriteInt32LittleEndian(destination, _count); + } + } + + private sealed class FlexibleAlgorithm : NonCryptographicHashAlgorithm + { + public bool IsReset { get; private set; } + public int GetCurrentHashCoreCallCount { get; private set; } + + public FlexibleAlgorithm(int hashLengthInBytes) + : base(hashLengthInBytes) + { + Reset(); + } + + public override void Append(ReadOnlySpan source) + { + if (source.Length > 0) + { + IsReset = false; + } + } + + public override void Reset() + { + IsReset = true; + } + + protected override void GetCurrentHashCore(Span destination) + { + Assert.Equal(HashLengthInBytes, destination.Length); + destination.Fill((byte)GetCurrentHashCoreCallCount); + GetCurrentHashCoreCallCount++; + } + } + + private sealed class FlexibleAlgorithmOverride : NonCryptographicHashAlgorithm + { + public bool IsReset { get; private set; } + public int GetCurrentHashCoreCallCount { get; private set; } + public int GetHashAndResetCoreCallCount { get; private set; } + + public FlexibleAlgorithmOverride(int hashLengthInBytes) + : base(hashLengthInBytes) + { + Reset(); + } + + public override void Append(ReadOnlySpan source) + { + if (source.Length > 0) + { + IsReset = false; + } + } + + public override void Reset() + { + IsReset = true; + } + + protected override void GetCurrentHashCore(Span destination) + { + Assert.Equal(HashLengthInBytes, destination.Length); + destination.Fill(0xFE); + GetCurrentHashCoreCallCount++; + } + + protected override void GetHashAndResetCore(Span destination) + { + Assert.Equal(HashLengthInBytes, destination.Length); + destination.Fill(0xFE); + Reset(); + GetHashAndResetCoreCallCount++; + } + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/NonCryptoHashTestDriver.cs b/src/libraries/System.IO.Hashing/tests/NonCryptoHashTestDriver.cs new file mode 100644 index 00000000000000..979e60a28065f6 --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/NonCryptoHashTestDriver.cs @@ -0,0 +1,361 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Reflection; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public abstract class NonCryptoHashTestDriver + { + private readonly int _hashLengthInBytes; + private readonly byte[] _emptyHash; + private string _emptyHashHex; + + protected NonCryptoHashTestDriver(byte[] emptyHash) + { + _hashLengthInBytes = emptyHash.Length; + _emptyHash = emptyHash; + } + + protected abstract NonCryptographicHashAlgorithm CreateInstance(); + + protected abstract byte[] StaticOneShot(byte[] source); + protected abstract byte[] StaticOneShot(ReadOnlySpan source); + protected abstract int StaticOneShot(ReadOnlySpan source, Span destination); + + protected abstract bool TryStaticOneShot( + ReadOnlySpan source, + Span destination, + out int bytesWritten); + + [Fact] + public void TestsDefined() + { + const string DriverSuffix = "Driver"; + Type implType = GetType(); + Type defType = typeof(NonCryptoHashTestDriver); + List? missingMethods = null; + + foreach (MethodInfo info in defType.GetMethods(BindingFlags.Instance | BindingFlags.NonPublic)) + { + if (info.IsFamily && info.Name.EndsWith(DriverSuffix, StringComparison.Ordinal)) + { + string targetMethodName = info.Name.Substring(0, info.Name.Length - DriverSuffix.Length); + + MethodInfo info2 = implType.GetMethod( + targetMethodName, + BindingFlags.Instance | BindingFlags.Public); + + if (info2 is null) + { + missingMethods ??= new List(); + missingMethods.Add(targetMethodName); + } + } + } + + if (missingMethods is not null) + { + Assert.Empty(missingMethods); + } + } + + [Fact] + public void VerifyLengthProperty() + { + NonCryptographicHashAlgorithm hash = CreateInstance(); + Assert.Equal(_hashLengthInBytes, hash.HashLengthInBytes); + } + + protected void InstanceAppendAllocateDriver(TestCase testCase) + { + NonCryptographicHashAlgorithm hash = CreateInstance(); + hash.Append(testCase.Input); + byte[] output = hash.GetCurrentHash(); + + Assert.Equal(testCase.OutputHex, TestCase.ToHexString(output)); + } + + protected void InstanceAppendAllocateAndResetDriver(TestCase testCase) + { + NonCryptographicHashAlgorithm hash = CreateInstance(); + hash.Append(testCase.Input); + byte[] output = hash.GetHashAndReset(); + + Assert.Equal(testCase.OutputHex, TestCase.ToHexString(output)); + + int written = hash.GetHashAndReset(output); + Assert.Equal(output.Length, written); + VerifyEmptyResult(output); + } + + protected void InstanceMultiAppendGetCurrentHashDriver(TestCase testCase) + { + ReadOnlySpan source = testCase.Input; + int div3 = source.Length / 3; + NonCryptographicHashAlgorithm hash = CreateInstance(); + hash.Append(source.Slice(0, div3)); + source = source.Slice(div3); + hash.Append(source.Slice(0, div3)); + source = source.Slice(div3); + hash.Append(source); + + Span buf = stackalloc byte[256]; + + // May as well check unaligned writes. + Span destination = buf.Slice(1); + int written = hash.GetCurrentHash(destination); + ReadOnlySpan answer = destination.Slice(0, written); + + testCase.VerifyResponse(answer); + + destination.Clear(); + hash.GetCurrentHash(destination); + testCase.VerifyResponse(answer); + } + + protected void InstanceVerifyEmptyStateDriver(TestCase testCase) + { + Span buf = stackalloc byte[256]; + NonCryptographicHashAlgorithm hash = CreateInstance(); + int written = hash.GetCurrentHash(buf); + VerifyEmptyResult(buf.Slice(0, written)); + + written = hash.GetHashAndReset(buf); + VerifyEmptyResult(buf.Slice(0, written)); + } + + protected void InstanceVerifyResetStateDriver(TestCase testCase) + { + NonCryptographicHashAlgorithm hash = CreateInstance(); + Span buf = stackalloc byte[233]; + int written = hash.GetHashAndReset(buf); + ReadOnlySpan ret = buf.Slice(0, written); + VerifyEmptyResult(ret); + + hash.Append(testCase.Input); + hash.Reset(); + hash.GetCurrentHash(buf); + VerifyEmptyResult(ret); + + // Manual call to Reset while already in a pristine state. + hash.Reset(); + hash.GetCurrentHash(buf); + VerifyEmptyResult(ret); + + hash.Append(testCase.Input); + hash.GetHashAndReset(buf); + testCase.VerifyResponse(ret); + + hash.GetHashAndReset(buf); + VerifyEmptyResult(ret); + } + + [Fact] + public void StaticOneShotNullArrayThrows() + { + AssertExtensions.Throws( + "source", + () => StaticOneShot((byte[])null)); + } + + protected void StaticVerifyOneShotArrayDriver(TestCase testCase) + { + byte[] answer = StaticOneShot(testCase.Input.ToArray()); + testCase.VerifyResponse(answer); + } + + protected void StaticVerifyOneShotSpanToArrayDriver(TestCase testCase) + { + byte[] answer = StaticOneShot(testCase.Input); + testCase.VerifyResponse(answer); + } + + protected void StaticVerifyOneShotSpanToSpanDriver(TestCase testCase) + { + Span destination = stackalloc byte[256]; + + int written = StaticOneShot(testCase.Input, destination); + testCase.VerifyResponse(destination.Slice(0, written)); + } + + protected void StaticVerifyTryOneShotDriver(TestCase testCase) + { + Span destination = stackalloc byte[256]; + + Assert.True(TryStaticOneShot(testCase.Input, destination, out int written)); + testCase.VerifyResponse(destination.Slice(0, written)); + } + + [Fact] + public void StaticVerifyOneShotSpanTooShortThrows() + { + byte[] destination = new byte[256]; + + for (int i = 0; i < _hashLengthInBytes; i++) + { + byte fill = (byte)~i; + destination.AsSpan().Fill(fill); + + AssertExtensions.Throws( + "destination", + () => StaticOneShot(ReadOnlySpan.Empty, destination.AsSpan(0, i))); + + for (int j = 0; j < destination.Length; j++) + { + Assert.Equal(fill, destination[j]); + } + } + } + + [Fact] + public void StaticVerifyOneShotSpanTooLongNoOverwrite() + { + Span buf = stackalloc byte[256]; + + for (int i = 10; i < 40; i++) + { + buf.Fill((byte)i); + + int written = StaticOneShot(buf.Slice(0, i), buf.Slice(i)); + Assert.Equal(_hashLengthInBytes, written); + + for (int j = i + written; j < buf.Length; j++) + { + Assert.Equal(i, buf[j]); + } + } + } + + [Fact] + public void StaticVerifyTryOneShotSpanTooLongNoOverwrite() + { + Span buf = stackalloc byte[256]; + + for (int i = 10; i < 40; i++) + { + buf.Fill((byte)i); + + Assert.True(TryStaticOneShot(buf.Slice(0, i), buf.Slice(i), out int written)); + Assert.Equal(_hashLengthInBytes, written); + + for (int j = i + written; j < buf.Length; j++) + { + Assert.Equal(i, buf[j]); + } + } + } + + [Fact] + public void StaticVerifyTryOneShotSpanTooShortNoWrites() + { + Span buf = stackalloc byte[256]; + + for (int i = 0; i < _hashLengthInBytes; i++) + { + byte fill = (byte)~i; + buf.Fill(fill); + + Assert.False(TryStaticOneShot(ReadOnlySpan.Empty, buf.Slice(0, i), out int written)); + Assert.Equal(0, written); + + for (int j = 0; j < buf.Length; j++) + { + Assert.Equal(fill, buf[j]); + } + } + } + + private void VerifyEmptyResult(ReadOnlySpan result) + { + if (!result.SequenceEqual(_emptyHash)) + { + // We know this will fail, but it gives a nice presentation. + + Assert.Equal( + _emptyHashHex ??= TestCase.ToHexString(_emptyHash), + TestCase.ToHexString(result)); + } + } + + public sealed class TestCase + { + private byte[] _input; + private byte[] _output; + + public string Name { get; } + public ReadOnlySpan Input => new ReadOnlySpan(_input); + public string OutputHex { get; } + + public TestCase(string name, byte[] input, byte[] output) + { + Name = name; + _input = input; + OutputHex = ToHexString(output); + _output = FromHexString(OutputHex); + } + + public TestCase(string name, byte[] input, string outputHex) + { + Name = name; + _input = input; + OutputHex = outputHex.ToUpperInvariant(); + _output = FromHexString(OutputHex); + } + + public TestCase(string name, string inputHex, string outputHex) + { + Name = name; + _input = FromHexString(inputHex); + OutputHex = outputHex.ToUpperInvariant(); + _output = FromHexString(OutputHex); + } + + internal void VerifyResponse(ReadOnlySpan response) + { + if (!response.SequenceEqual(_output)) + { + // We know this will fail, but it gives a nice presentation. + Assert.Equal(OutputHex, ToHexString(response)); + } + } + + internal static string ToHexString(ReadOnlySpan input) + { +#if NET5_0_OR_GREATER + return Convert.ToHexString(input); +#else + var builder = new global::System.Text.StringBuilder(input.Length * 2); + + foreach (byte b in input) + { + builder.Append(b.ToString("X2")); + } + + return builder.ToString(); +#endif + } + + internal static byte[] FromHexString(string hexString) + { +#if NET5_0_OR_GREATER + return Convert.FromHexString(hexString); +#else + byte[] bytes = new byte[hexString.Length / 2]; + + for (int i = 0; i < hexString.Length; i += 2) + { + string s = hexString.Substring(i, 2); + bytes[i / 2] = byte.Parse(s, global::System.Globalization.NumberStyles.HexNumber, null); + } + + return bytes; +#endif + } + + public override string ToString() => Name; + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj new file mode 100644 index 00000000000000..efd86050b39015 --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj @@ -0,0 +1,17 @@ + + + true + $(NetCoreAppCurrent);net461 + + + + + + + CommonTest\System\IO\PositionValueStream.cs + + + + + + From 4c0e824814eee99347d2e12874047eba3c6655a5 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Fri, 28 May 2021 14:40:02 -0700 Subject: [PATCH 02/18] Add CRC-64(-ECMA) --- .../ref/System.IO.Hashing.cs | 12 ++ .../src/System.IO.Hashing.csproj | 2 + .../src/System/IO/Hashing/Crc64.Table.cs | 37 ++++ .../src/System/IO/Hashing/Crc64.cs | 166 ++++++++++++++++++ .../System.IO.Hashing/tests/Crc64Tests.cs | 146 +++++++++++++++ .../tests/System.IO.Hashing.Tests.csproj | 1 + 6 files changed, 364 insertions(+) create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.Table.cs create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.cs create mode 100644 src/libraries/System.IO.Hashing/tests/Crc64Tests.cs diff --git a/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs index eedadf5ad1725b..253875e4f8b55f 100644 --- a/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs +++ b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs @@ -18,6 +18,18 @@ protected override void GetHashAndResetCore(System.Span destination) { } public override void Reset() { } public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } } + public sealed partial class Crc64 : System.IO.Hashing.NonCryptographicHashAlgorithm + { + public Crc64() : base (default(int)) { } + public override void Append(System.ReadOnlySpan source) { } + protected override void GetCurrentHashCore(System.Span destination) { } + protected override void GetHashAndResetCore(System.Span destination) { } + public static byte[] Hash(byte[] source) { throw null; } + public static byte[] Hash(System.ReadOnlySpan source) { throw null; } + public static int Hash(System.ReadOnlySpan source, System.Span destination) { throw null; } + public override void Reset() { } + public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } + } public abstract partial class NonCryptographicHashAlgorithm { protected NonCryptographicHashAlgorithm(int hashLengthInBytes) { } diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj index 4766f6d027d8ad..fa160f9e255580 100644 --- a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -9,6 +9,8 @@ + + diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.Table.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.Table.cs new file mode 100644 index 00000000000000..431c0b89a6f6ad --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.Table.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.IO.Hashing +{ + public sealed partial class Crc64 : NonCryptographicHashAlgorithm + { + // Pre-computed CRC-64 transition table. + private static readonly ulong[] s_crcLookup = GenerateTable(0x42F0E1EBA9EA3693); + + private static ulong[] GenerateTable(ulong polynomial) + { + ulong[] table = new ulong[256]; + + for (int i = 0; i < 256; i++) + { + ulong val = unchecked((ulong)i) << 56; + + for (int j = 0; j < 8; j++) + { + if ((val & 0x8000_0000_0000_0000) == 0) + { + val <<= 1; + } + else + { + val = (val << 1) ^ polynomial; + } + } + + table[i] = val; + } + + return table; + } + } +} diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.cs new file mode 100644 index 00000000000000..ca418218259790 --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.cs @@ -0,0 +1,166 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers.Binary; + +namespace System.IO.Hashing +{ + /// + /// Provides an implementation of the CRC-64 algorithm as described in ECMA-182, Annex B. + /// + /// + /// + /// This implementation emits the answer in the Big Endian byte order so that + /// the CRC residue relationship (CRC(message concat CRC(message))) is a fixed value) holds. + /// For CRC-64 this stable output is the byte sequence + /// { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }. + /// + /// + /// There are multiple, incompatible, definitions of a 64-bit cyclic redundancy + /// check (CRC) algorithm. When interoperating with another system, ensure that you + /// are using the same definition. The definition used by this implementation is not + /// compatible with the cyclic redundancy check described in ISO 3309. + /// + /// + public sealed partial class Crc64 : NonCryptographicHashAlgorithm + { + private const ulong InitialState = 0UL; + private const int Size = sizeof(ulong); + + private ulong _crc = InitialState; + + /// + /// Initializes a new instance of the class. + /// + public Crc64() + : base(Size) + { + } + + /// + /// Appends the contents of to the data already + /// processed for the current hash computation. + /// + /// The data to process. + public override void Append(ReadOnlySpan source) + { + _crc = Update(_crc, source); + } + + /// + /// Resets the hash computation to the initial state. + /// + public override void Reset() + { + _crc = InitialState; + } + + /// + /// Writes the computed hash value to + /// without modifying accumulated state. + /// + /// The buffer that receives the computed hash value. + protected override void GetCurrentHashCore(Span destination) + { + BinaryPrimitives.WriteUInt64BigEndian(destination, _crc); + } + + /// + /// Writes the computed hash value to + /// then clears the accumulated state. + /// + protected override void GetHashAndResetCore(Span destination) + { + BinaryPrimitives.WriteUInt64BigEndian(destination, _crc); + _crc = InitialState; + } + + /// + /// Computes the CRC-64 hash of the provided data. + /// + /// The data to hash. + /// The CRC-64 hash of the provided data. + /// + /// is . + /// + public static byte[] Hash(byte[] source) + { + if (source is null) + throw new ArgumentNullException(nameof(source)); + + return Hash(new ReadOnlySpan(source)); + } + + /// + /// Computes the CRC-64 hash of the provided data. + /// + /// The data to hash. + /// The CRC-64 hash of the provided data. + public static byte[] Hash(ReadOnlySpan source) + { + byte[] ret = new byte[Size]; + StaticHash(source, ret); + return ret; + } + + /// + /// Attempts to compute the CRC-64 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// On success, receives the number of bytes written to . + /// + /// + /// if is long enough to receive + /// the computed hash value (8 bytes); otherwise, . + /// + public static bool TryHash(ReadOnlySpan source, Span destination, out int bytesWritten) + { + if (destination.Length < Size) + { + bytesWritten = 0; + return false; + } + + bytesWritten = StaticHash(source, destination); + return true; + } + + /// + /// Computes the CRC-64 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// The number of bytes written to . + /// + public static int Hash(ReadOnlySpan source, Span destination) + { + if (destination.Length < Size) + throw new ArgumentException(SR.Argument_DestinationTooShort, nameof(destination)); + + return StaticHash(source, destination); + } + + private static int StaticHash(ReadOnlySpan source, Span destination) + { + ulong crc = InitialState; + crc = Update(crc, source); + BinaryPrimitives.WriteUInt64BigEndian(destination, crc); + return Size; + } + + private static ulong Update(ulong crc, ReadOnlySpan source) + { + for (int i = 0; i < source.Length; i++) + { + ulong idx = (crc >> 56); + idx ^= source[i]; + crc = s_crcLookup[idx] ^ (crc << 8); + } + + return crc; + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/Crc64Tests.cs b/src/libraries/System.IO.Hashing/tests/Crc64Tests.cs new file mode 100644 index 00000000000000..1a838015723d7e --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/Crc64Tests.cs @@ -0,0 +1,146 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public class Crc64Tests : NonCryptoHashTestDriver + { + public Crc64Tests() : base(new byte[8]) + { + } + + public static IEnumerable TestCases + { + get + { + object[] arr = new object[1]; + + foreach (TestCase testCase in TestCaseDefinitions) + { + arr[0] = testCase; + yield return arr; + } + } + } + + protected static IEnumerable TestCaseDefinitions { get; } = + new[] + { + new TestCase( + "Empty", + "", + "0000000000000000"), + new TestCase( + "One", + "01", + "42F0E1EBA9EA3693"), + // Because CRC-64 has an initial state of 0, any input that is all + // zero-valued bytes will always produce 0x0ul as the output, since it + // never leaves state 0. + new TestCase( + "{ 0x00 }", + "00", + "0000000000000000"), + // Once it has left the initial state, zero-value bytes matter. + new TestCase( + "{ 0x01, 0x00 }", + "0100", + "AF052A6B538EDF09"), + new TestCase( + "Zero-Residue", + "0000000000000000", + "0000000000000000"), + new TestCase( + "Self-test 123456789", + Encoding.ASCII.GetBytes("123456789"), + "6C40DF5F0B497347"), + new TestCase( + "Self-test residue", + "3132333435363738396C40DF5F0B497347", + "0000000000000000"), + new TestCase( + "The quick brown fox jumps over the lazy dog", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog"), + "41E05242FFA9883B"), + }; + + protected override NonCryptographicHashAlgorithm CreateInstance() => new Crc64(); + + protected override byte[] StaticOneShot(byte[] source) => Crc64.Hash(source); + + protected override byte[] StaticOneShot(ReadOnlySpan source) => Crc64.Hash(source); + + protected override int StaticOneShot(ReadOnlySpan source, Span destination) => + Crc64.Hash(source, destination); + + protected override bool TryStaticOneShot(ReadOnlySpan source, Span destination, out int bytesWritten) => + Crc64.TryHash(source, destination, out bytesWritten); + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocate(TestCase testCase) + { + InstanceAppendAllocateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocateAndReset(TestCase testCase) + { + InstanceAppendAllocateAndResetDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceMultiAppendGetCurrentHash(TestCase testCase) + { + InstanceMultiAppendGetCurrentHashDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyEmptyState(TestCase testCase) + { + InstanceVerifyEmptyStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyResetState(TestCase testCase) + { + InstanceVerifyResetStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotArray(TestCase testCase) + { + StaticVerifyOneShotArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToArray(TestCase testCase) + { + StaticVerifyOneShotSpanToArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToSpan(TestCase testCase) + { + StaticVerifyOneShotSpanToSpanDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyTryOneShot(TestCase testCase) + { + StaticVerifyTryOneShotDriver(testCase); + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj index efd86050b39015..9cc3688843d54c 100644 --- a/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj +++ b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj @@ -5,6 +5,7 @@ + From e7f1b8259c110666e90883dfe11f623d4ee56d88 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Fri, 28 May 2021 16:01:26 -0700 Subject: [PATCH 03/18] Change test empty hash initializer style --- src/libraries/System.IO.Hashing/tests/Crc32Tests.cs | 5 ++++- src/libraries/System.IO.Hashing/tests/Crc64Tests.cs | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs b/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs index c8f5486c89b17b..6d2993f1509e47 100644 --- a/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs +++ b/src/libraries/System.IO.Hashing/tests/Crc32Tests.cs @@ -9,7 +9,10 @@ namespace System.IO.Hashing.Tests { public class Crc32Tests : NonCryptoHashTestDriver { - public Crc32Tests() : base(new byte[4]) + private static readonly byte[] s_emptyHashValue = new byte[4]; + + public Crc32Tests() + : base(s_emptyHashValue) { } diff --git a/src/libraries/System.IO.Hashing/tests/Crc64Tests.cs b/src/libraries/System.IO.Hashing/tests/Crc64Tests.cs index 1a838015723d7e..d647e4b9587989 100644 --- a/src/libraries/System.IO.Hashing/tests/Crc64Tests.cs +++ b/src/libraries/System.IO.Hashing/tests/Crc64Tests.cs @@ -9,7 +9,10 @@ namespace System.IO.Hashing.Tests { public class Crc64Tests : NonCryptoHashTestDriver { - public Crc64Tests() : base(new byte[8]) + private static readonly byte[] s_emptyHashValue = new byte[8]; + + public Crc64Tests() + : base(s_emptyHashValue) { } From f959a16564171e0c63b7d8874cdb75b90d26edd9 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Fri, 28 May 2021 17:21:23 -0700 Subject: [PATCH 04/18] Add XxHash32 --- .../ref/System.IO.Hashing.cs | 11 ++ .../src/System.IO.Hashing.csproj | 3 + .../src/System/IO/Hashing/BitOperations.cs | 14 ++ .../src/System/IO/Hashing/XxHash32.State.cs | 113 +++++++++++ .../src/System/IO/Hashing/XxHash32.cs | 182 ++++++++++++++++++ .../src/System/IO/Hashing/XxHash64.cs | 0 .../tests/System.IO.Hashing.Tests.csproj | 1 + .../System.IO.Hashing/tests/XxHash32Tests.cs | 170 ++++++++++++++++ 8 files changed, 494 insertions(+) create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/BitOperations.cs create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs create mode 100644 src/libraries/System.IO.Hashing/tests/XxHash32Tests.cs diff --git a/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs index 253875e4f8b55f..7546d8f17fac6e 100644 --- a/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs +++ b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs @@ -49,4 +49,15 @@ protected virtual void GetHashAndResetCore(System.Span destination) { } public bool TryGetCurrentHash(System.Span destination, out int bytesWritten) { throw null; } public bool TryGetHashAndReset(System.Span destination, out int bytesWritten) { throw null; } } + public sealed partial class XxHash32 : System.IO.Hashing.NonCryptographicHashAlgorithm + { + public XxHash32() : base (default(int)) { } + public override void Append(System.ReadOnlySpan source) { } + protected override void GetCurrentHashCore(System.Span destination) { } + public static byte[] Hash(byte[] source) { throw null; } + public static byte[] Hash(System.ReadOnlySpan source) { throw null; } + public static int Hash(System.ReadOnlySpan source, System.Span destination) { throw null; } + public override void Reset() { } + public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } + } } diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj index fa160f9e255580..32eab103853350 100644 --- a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -7,10 +7,13 @@ true + + + diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/BitOperations.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/BitOperations.cs new file mode 100644 index 00000000000000..83b3397c06c833 --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/BitOperations.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System.IO.Hashing +{ + internal static class BitOperations + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint RotateLeft(uint value, int offset) + => (value << offset) | (value >> (32 - offset)); + } +} diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs new file mode 100644 index 00000000000000..ed10b41e39e55c --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs @@ -0,0 +1,113 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers.Binary; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +// Implemented from the specification at +// https://github.com/Cyan4973/xxHash/blob/f9155bd4c57e2270a4ffbb176485e5d713de1c9b/doc/xxhash_spec.md + +namespace System.IO.Hashing +{ + public sealed partial class XxHash32 + { + private struct State + { + private const uint Prime32_1 = 0x9E3779B1; + private const uint Prime32_2 = 0x85EBCA77; + private const uint Prime32_3 = 0xC2B2AE3D; + private const uint Prime32_4 = 0x27D4EB2F; + private const uint Prime32_5 = 0x165667B1; + + private uint _acc1; + private uint _acc2; + private uint _acc3; + private uint _acc4; + private readonly uint _smallAcc; + + internal State(uint seed) + { + unchecked + { + _acc1 = seed + Prime32_1 + Prime32_2; + _acc2 = seed + Prime32_2; + _acc3 = seed; + _acc4 = seed - Prime32_1; + + _smallAcc = seed + Prime32_5; + } + } + + internal void ProcessStripe(ReadOnlySpan source) + { + Debug.Assert(source.Length >= StripeSize); + + _acc1 = ApplyRound(_acc1, source); + _acc2 = ApplyRound(_acc2, source.Slice(sizeof(uint))); + _acc3 = ApplyRound(_acc3, source.Slice(2 * sizeof(uint))); + _acc4 = ApplyRound(_acc4, source.Slice(3 * sizeof(uint))); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private readonly uint Converge() + { + return + BitOperations.RotateLeft(_acc1, 1) + + BitOperations.RotateLeft(_acc2, 7) + + BitOperations.RotateLeft(_acc3, 12) + + BitOperations.RotateLeft(_acc4, 18); + } + + private static uint ApplyRound(uint acc, ReadOnlySpan lane) + { + unchecked + { + acc += BinaryPrimitives.ReadUInt32LittleEndian(lane) * Prime32_2; + acc = BitOperations.RotateLeft(acc, 13); + acc *= Prime32_1; + } + + return acc; + } + + internal readonly uint Complete(int length, ReadOnlySpan remaining) + { + unchecked + { + uint acc = length >= StripeSize ? Converge() : _smallAcc; + + acc += (uint)length; + + while (remaining.Length >= sizeof(uint)) + { + uint lane = BinaryPrimitives.ReadUInt32LittleEndian(remaining); + acc += lane * Prime32_3; + acc = BitOperations.RotateLeft(acc, 17); + acc *= Prime32_4; + + remaining = remaining.Slice(sizeof(uint)); + } + + while (remaining.Length > 0) + { + uint lane = remaining[0]; + acc += lane * Prime32_5; + acc = BitOperations.RotateLeft(acc, 11); + acc *= Prime32_1; + + remaining = remaining.Slice(1); + } + + acc ^= (acc >> 15); + acc *= Prime32_2; + acc ^= (acc >> 13); + acc *= Prime32_3; + acc ^= (acc >> 16); + + return acc; + } + } + } + } +} diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs new file mode 100644 index 00000000000000..47fbba78338fcb --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers.Binary; + +// Implemented from the specification at +// https://github.com/Cyan4973/xxHash/blob/f9155bd4c57e2270a4ffbb176485e5d713de1c9b/doc/xxhash_spec.md + +namespace System.IO.Hashing +{ + /// + /// Provides an implementation of the XxHash32 algorithm. + /// + public sealed partial class XxHash32 : NonCryptographicHashAlgorithm + { + private const int HashSize = sizeof(uint); + private const int StripeSize = 4 * sizeof(uint); + + private State _state; + private byte[] _holdback; + private int _length; + + /// + /// Initializes a new instance of the class. + /// + public XxHash32() + : base(HashSize) + { + Reset(); + } + + public override void Reset() + { + _state = new State(0); + _length = 0; + } + + public override void Append(ReadOnlySpan source) + { + // Every time we've read 16 bytes, process the stripe. + // Data that isn't perfectly mod-16 gets stored in a holdback + // buffer. + + int held = _length & 0x0F; + + if (held != 0) + { + int remain = StripeSize - held; + + if (source.Length > remain) + { + source.Slice(0, remain).CopyTo(_holdback.AsSpan(held)); + _state.ProcessStripe(_holdback); + + source = source.Slice(remain); + _length += remain; + } + else + { + source.CopyTo(_holdback.AsSpan(held)); + _length += source.Length; + return; + } + } + + while (source.Length >= StripeSize) + { + _state.ProcessStripe(source); + source = source.Slice(StripeSize); + _length += StripeSize; + } + + if (source.Length > 0) + { + _holdback ??= new byte[StripeSize]; + source.CopyTo(_holdback); + _length += source.Length; + } + } + + protected override void GetCurrentHashCore(Span destination) + { + unchecked + { + int remainingLength = _length & 0x0F; + ReadOnlySpan remaining = ReadOnlySpan.Empty; + + if (remainingLength > 0) + { + remaining = new ReadOnlySpan(_holdback, 0, remainingLength); + } + + uint acc = _state.Complete(_length, remaining); + BinaryPrimitives.WriteUInt32BigEndian(destination, acc); + } + } + + /// + /// Computes the XxHash32 hash of the provided data. + /// + /// The data to hash. + /// The XxHash32 hash of the provided data. + /// + /// is . + /// + public static byte[] Hash(byte[] source) + { + if (source is null) + throw new ArgumentNullException(nameof(source)); + + return Hash(new ReadOnlySpan(source)); + } + + /// + /// Computes the XxHash32 hash of the provided data. + /// + /// The data to hash. + /// The XxHash32 hash of the provided data. + public static byte[] Hash(ReadOnlySpan source) + { + byte[] ret = new byte[HashSize]; + StaticHash(source, ret); + return ret; + } + + /// + /// Attempts to compute the XxHash32 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// On success, receives the number of bytes written to . + /// + /// + /// if is long enough to receive + /// the computed hash value (4 bytes); otherwise, . + /// + public static bool TryHash(ReadOnlySpan source, Span destination, out int bytesWritten) + { + if (destination.Length < HashSize) + { + bytesWritten = 0; + return false; + } + + bytesWritten = StaticHash(source, destination); + return true; + } + + /// + /// Computes the XxHash32 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// The number of bytes written to . + /// + public static int Hash(ReadOnlySpan source, Span destination) + { + if (destination.Length < HashSize) + throw new ArgumentException(SR.Argument_DestinationTooShort, nameof(destination)); + + return StaticHash(source, destination); + } + + private static int StaticHash(ReadOnlySpan source, Span destination) + { + int totalLength = source.Length; + State state = new State(0); + + while (source.Length > StripeSize) + { + state.ProcessStripe(source); + source = source.Slice(StripeSize); + } + + uint val = state.Complete(totalLength, source); + BinaryPrimitives.WriteUInt32BigEndian(destination, val); + return HashSize; + } + } +} diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj index 9cc3688843d54c..d7f040d2f977ee 100644 --- a/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj +++ b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj @@ -8,6 +8,7 @@ + CommonTest\System\IO\PositionValueStream.cs diff --git a/src/libraries/System.IO.Hashing/tests/XxHash32Tests.cs b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.cs new file mode 100644 index 00000000000000..f9dccad0294028 --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.cs @@ -0,0 +1,170 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public class XxHash32Tests : NonCryptoHashTestDriver + { + private static readonly byte[] s_emptyHashValue = new byte[] { 0x02, 0xCC, 0x5D, 0x05 }; + + public XxHash32Tests() + : base(s_emptyHashValue) + { + } + + public static IEnumerable TestCases + { + get + { + object[] arr = new object[1]; + + foreach (TestCase testCase in TestCaseDefinitions) + { + arr[0] = testCase; + yield return arr; + } + } + } + + private const string DotNetHashesThis = ".NET Hashes This!"; + private const string DotNetHashesThis3 = DotNetHashesThis + DotNetHashesThis + DotNetHashesThis; + private const string DotNetNCHashing = ".NET now has non-crypto hashing"; + private const string DotNetNCHashing3 = DotNetNCHashing + DotNetNCHashing + DotNetNCHashing; + + protected static IEnumerable TestCaseDefinitions { get; } = + new[] + { + //https://asecuritysite.com/encryption/xxHash, Example 1 + new TestCase( + "Nobody inspects the spammish repetition", + Encoding.ASCII.GetBytes("Nobody inspects the spammish repetition"), + "E2293B2F"), + //https://asecuritysite.com/encryption/xxHash, Example 2 + new TestCase( + "The quick brown fox jumps over the lazy dog", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog"), + "E85EA4DE"), + //https://asecuritysite.com/encryption/xxHash, Example 3 + new TestCase( + "The quick brown fox jumps over the lazy dog.", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog."), + "68D039C8"), + + // Manual exploration to force boundary conditions in code coverage. + // Output values produced by existing tools. + + // The "in three pieces" test causes this to build up in the accumulator every time. + new TestCase( + "abc", + Encoding.ASCII.GetBytes("abc"), + "32D153FF"), + // Accumulates every time. + new TestCase( + "123456", + "313233343536", + "B7014066"), + // In the 3 pieces test, the third call to Append fills and process the holdback, + // then stores the remainder. + new TestCase( + "12345678901234567890", + "3132333435363738393031323334353637383930", + "2D0C3D1B"), + // Same as above, but ends up with a byte that doesn't align to uints. + new TestCase( + "123456789012345678901", + "313233343536373839303132333435363738393031", + "8ED1B04E"), + // 17 * 3 bytes long, in the "in three pieces" test each call to Append processes + // a lane and holds one byte. + new TestCase( + $"{DotNetHashesThis} (x3)", + Encoding.ASCII.GetBytes(DotNetHashesThis3), + "1FE08A04"), + // 31 * 3 bytes long. In the "in three pieces" test the later calls to Append call + // into ProcessStripe with unaligned starts. + new TestCase( + $"{DotNetNCHashing} (x3)", + Encoding.ASCII.GetBytes(DotNetNCHashing3), + "65242024"), + }; + + protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash32(); + + protected override byte[] StaticOneShot(byte[] source) => XxHash32.Hash(source); + + protected override byte[] StaticOneShot(ReadOnlySpan source) => XxHash32.Hash(source); + + protected override int StaticOneShot(ReadOnlySpan source, Span destination) => + XxHash32.Hash(source, destination); + + protected override bool TryStaticOneShot(ReadOnlySpan source, Span destination, out int bytesWritten) => + XxHash32.TryHash(source, destination, out bytesWritten); + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocate(TestCase testCase) + { + InstanceAppendAllocateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocateAndReset(TestCase testCase) + { + InstanceAppendAllocateAndResetDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceMultiAppendGetCurrentHash(TestCase testCase) + { + InstanceMultiAppendGetCurrentHashDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyEmptyState(TestCase testCase) + { + InstanceVerifyEmptyStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyResetState(TestCase testCase) + { + InstanceVerifyResetStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotArray(TestCase testCase) + { + StaticVerifyOneShotArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToArray(TestCase testCase) + { + StaticVerifyOneShotSpanToArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToSpan(TestCase testCase) + { + StaticVerifyOneShotSpanToSpanDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyTryOneShot(TestCase testCase) + { + StaticVerifyTryOneShotDriver(testCase); + } + } +} From d56e84e15de61b28a538b7f9cb230539749dcac3 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Tue, 1 Jun 2021 14:42:10 -0700 Subject: [PATCH 05/18] Add XxHash64 --- .../ref/System.IO.Hashing.cs | 11 ++ .../src/System.IO.Hashing.csproj | 2 + .../src/System/IO/Hashing/BitOperations.cs | 4 + .../src/System/IO/Hashing/XxHash64.State.cs | 151 +++++++++++++++ .../src/System/IO/Hashing/XxHash64.cs | 182 +++++++++++++++++ .../tests/System.IO.Hashing.Tests.csproj | 1 + .../System.IO.Hashing/tests/XxHash64Tests.cs | 183 ++++++++++++++++++ 7 files changed, 534 insertions(+) create mode 100644 src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs create mode 100644 src/libraries/System.IO.Hashing/tests/XxHash64Tests.cs diff --git a/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs index 7546d8f17fac6e..bc0f746b83106e 100644 --- a/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs +++ b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs @@ -60,4 +60,15 @@ protected override void GetCurrentHashCore(System.Span destination) { } public override void Reset() { } public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } } + public sealed partial class XxHash64 : System.IO.Hashing.NonCryptographicHashAlgorithm + { + public XxHash64() : base (default(int)) { } + public override void Append(System.ReadOnlySpan source) { } + protected override void GetCurrentHashCore(System.Span destination) { } + public static byte[] Hash(byte[] source) { throw null; } + public static byte[] Hash(System.ReadOnlySpan source) { throw null; } + public static int Hash(System.ReadOnlySpan source, System.Span destination) { throw null; } + public override void Reset() { } + public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } + } } diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj index 32eab103853350..6ea7333e802bd2 100644 --- a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -14,6 +14,8 @@ + + diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/BitOperations.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/BitOperations.cs index 83b3397c06c833..9e0c9845dd36f5 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/BitOperations.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/BitOperations.cs @@ -10,5 +10,9 @@ internal static class BitOperations [MethodImpl(MethodImplOptions.AggressiveInlining)] public static uint RotateLeft(uint value, int offset) => (value << offset) | (value >> (32 - offset)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ulong RotateLeft(ulong value, int offset) + => (value << offset) | (value >> (64 - offset)); } } diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs new file mode 100644 index 00000000000000..912f422cfe8cb3 --- /dev/null +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs @@ -0,0 +1,151 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers.Binary; +using System.Diagnostics; + +// Implemented from the specification at +// https://github.com/Cyan4973/xxHash/blob/f9155bd4c57e2270a4ffbb176485e5d713de1c9b/doc/xxhash_spec.md + +namespace System.IO.Hashing +{ + public sealed partial class XxHash64 + { + private struct State + { + private const ulong Prime64_1 = 0x9E3779B185EBCA87; + private const ulong Prime64_2 = 0xC2B2AE3D27D4EB4F; + private const ulong Prime64_3 = 0x165667B19E3779F9; + private const ulong Prime64_4 = 0x85EBCA77C2B2AE63; + private const ulong Prime64_5 = 0x27D4EB2F165667C5; + + private ulong _acc1; + private ulong _acc2; + private ulong _acc3; + private ulong _acc4; + private readonly ulong _smallAcc; + + internal State(ulong seed) + { + unchecked + { + _acc1 = seed + Prime64_1 + Prime64_2; + _acc2 = seed + Prime64_2; + _acc3 = seed; + _acc4 = seed - Prime64_1; + + _smallAcc = seed + Prime64_5; + } + } + + internal void ProcessStripe(ReadOnlySpan source) + { + Debug.Assert(source.Length >= StripeSize); + + _acc1 = ApplyRound(_acc1, source); + _acc2 = ApplyRound(_acc2, source.Slice(sizeof(ulong))); + _acc3 = ApplyRound(_acc3, source.Slice(2 * sizeof(ulong))); + _acc4 = ApplyRound(_acc4, source.Slice(3 * sizeof(ulong))); + } + + private static ulong MergeAccumulator(ulong acc, ulong accN) + { + unchecked + { + acc ^= ApplyRound(0, accN); + acc *= Prime64_1; + acc += Prime64_4; + } + + return acc; + } + + private readonly ulong Converge() + { + unchecked + { + ulong acc = + BitOperations.RotateLeft(_acc1, 1) + + BitOperations.RotateLeft(_acc2, 7) + + BitOperations.RotateLeft(_acc3, 12) + + BitOperations.RotateLeft(_acc4, 18); + + acc = MergeAccumulator(acc, _acc1); + acc = MergeAccumulator(acc, _acc2); + acc = MergeAccumulator(acc, _acc3); + acc = MergeAccumulator(acc, _acc4); + + return acc; + } + } + + private static ulong ApplyRound(ulong acc, ReadOnlySpan lane) + { + return ApplyRound(acc, BinaryPrimitives.ReadUInt64LittleEndian(lane)); + } + + private static ulong ApplyRound(ulong acc, ulong lane) + { + unchecked + { + acc += lane * Prime64_2; + acc = BitOperations.RotateLeft(acc, 31); + acc *= Prime64_1; + } + + return acc; + } + + internal readonly ulong Complete(int length, ReadOnlySpan remaining) + { + unchecked + { + ulong acc = length >= StripeSize ? Converge() : _smallAcc; + + acc += (ulong)length; + + while (remaining.Length >= sizeof(ulong)) + { + ulong lane = BinaryPrimitives.ReadUInt64LittleEndian(remaining); + acc ^= ApplyRound(0, lane); + acc = BitOperations.RotateLeft(acc, 27); + acc *= Prime64_1; + acc += Prime64_4; + + remaining = remaining.Slice(sizeof(ulong)); + } + + // Doesn't need to be a while since it can occur at most once. + if (remaining.Length >= sizeof(uint)) + { + ulong lane = BinaryPrimitives.ReadUInt32LittleEndian(remaining); + acc ^= lane * Prime64_1; + acc = BitOperations.RotateLeft(acc, 23); + acc *= Prime64_2; + acc += Prime64_3; + + remaining = remaining.Slice(sizeof(uint)); + } + + while (remaining.Length > 0) + { + ulong lane = remaining[0]; + acc ^= lane * Prime64_5; + acc = BitOperations.RotateLeft(acc, 11); + acc *= Prime64_1; + + remaining = remaining.Slice(1); + } + + acc ^= (acc >> 33); + acc *= Prime64_2; + acc ^= (acc >> 29); + acc *= Prime64_3; + acc ^= (acc >> 32); + + return acc; + } + } + } + } +} diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs index e69de29bb2d1d6..70a6623ede7dcc 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers.Binary; + +// Implemented from the specification at +// https://github.com/Cyan4973/xxHash/blob/f9155bd4c57e2270a4ffbb176485e5d713de1c9b/doc/xxhash_spec.md + +namespace System.IO.Hashing +{ + /// + /// Provides an implementation of the XxHash64 algorithm. + /// + public sealed partial class XxHash64 : NonCryptographicHashAlgorithm + { + private const int HashSize = sizeof(ulong); + private const int StripeSize = 4 * sizeof(ulong); + + private State _state; + private byte[] _holdback; + private int _length; + + /// + /// Initializes a new instance of the class. + /// + public XxHash64() + : base(HashSize) + { + Reset(); + } + + public override void Reset() + { + _state = new State(0); + _length = 0; + } + + public override void Append(ReadOnlySpan source) + { + // Every time we've read 16 bytes, process the stripe. + // Data that isn't perfectly mod-16 gets stored in a holdback + // buffer. + + int held = _length & 0x1F; + + if (held != 0) + { + int remain = StripeSize - held; + + if (source.Length > remain) + { + source.Slice(0, remain).CopyTo(_holdback.AsSpan(held)); + _state.ProcessStripe(_holdback); + + source = source.Slice(remain); + _length += remain; + } + else + { + source.CopyTo(_holdback.AsSpan(held)); + _length += source.Length; + return; + } + } + + while (source.Length >= StripeSize) + { + _state.ProcessStripe(source); + source = source.Slice(StripeSize); + _length += StripeSize; + } + + if (source.Length > 0) + { + _holdback ??= new byte[StripeSize]; + source.CopyTo(_holdback); + _length += source.Length; + } + } + + protected override void GetCurrentHashCore(Span destination) + { + unchecked + { + int remainingLength = _length & 0x1F; + ReadOnlySpan remaining = ReadOnlySpan.Empty; + + if (remainingLength > 0) + { + remaining = new ReadOnlySpan(_holdback, 0, remainingLength); + } + + ulong acc = _state.Complete(_length, remaining); + BinaryPrimitives.WriteUInt64BigEndian(destination, acc); + } + } + + /// + /// Computes the XxHash64 hash of the provided data. + /// + /// The data to hash. + /// The XxHash64 hash of the provided data. + /// + /// is . + /// + public static byte[] Hash(byte[] source) + { + if (source is null) + throw new ArgumentNullException(nameof(source)); + + return Hash(new ReadOnlySpan(source)); + } + + /// + /// Computes the XxHash64 hash of the provided data. + /// + /// The data to hash. + /// The XxHash64 hash of the provided data. + public static byte[] Hash(ReadOnlySpan source) + { + byte[] ret = new byte[HashSize]; + StaticHash(source, ret); + return ret; + } + + /// + /// Attempts to compute the XxHash64 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// On success, receives the number of bytes written to . + /// + /// + /// if is long enough to receive + /// the computed hash value (4 bytes); otherwise, . + /// + public static bool TryHash(ReadOnlySpan source, Span destination, out int bytesWritten) + { + if (destination.Length < HashSize) + { + bytesWritten = 0; + return false; + } + + bytesWritten = StaticHash(source, destination); + return true; + } + + /// + /// Computes the XxHash64 hash of the provided data into the provided destination. + /// + /// The data to hash. + /// The buffer that receives the computed hash value. + /// + /// The number of bytes written to . + /// + public static int Hash(ReadOnlySpan source, Span destination) + { + if (destination.Length < HashSize) + throw new ArgumentException(SR.Argument_DestinationTooShort, nameof(destination)); + + return StaticHash(source, destination); + } + + private static int StaticHash(ReadOnlySpan source, Span destination) + { + int totalLength = source.Length; + State state = new State(0); + + while (source.Length > StripeSize) + { + state.ProcessStripe(source); + source = source.Slice(StripeSize); + } + + ulong val = state.Complete(totalLength, source); + BinaryPrimitives.WriteUInt64BigEndian(destination, val); + return HashSize; + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj index d7f040d2f977ee..e60fbcfe0ea69f 100644 --- a/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj +++ b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj @@ -9,6 +9,7 @@ + CommonTest\System\IO\PositionValueStream.cs diff --git a/src/libraries/System.IO.Hashing/tests/XxHash64Tests.cs b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.cs new file mode 100644 index 00000000000000..973d108fc93778 --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.cs @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public class XxHash64Tests : NonCryptoHashTestDriver + { + private static readonly byte[] s_emptyHashValue = + new byte[] { 0xEF, 0x46, 0xDB, 0x37, 0x51, 0xD8, 0xE9, 0x99 }; + + public XxHash64Tests() + : base(s_emptyHashValue) + { + } + + public static IEnumerable TestCases + { + get + { + object[] arr = new object[1]; + + foreach (TestCase testCase in TestCaseDefinitions) + { + arr[0] = testCase; + yield return arr; + } + } + } + + private const string ThirtyThreeBytes = "This string has 33 ASCII bytes..."; + private const string ThirtyThreeBytes3 = ThirtyThreeBytes + ThirtyThreeBytes + ThirtyThreeBytes; + private const string DotNetNCHashing = ".NET now has non-crypto hashing"; + private const string SixtyThreeBytes = "A sixty-three byte test input requires substantial forethought!"; + private const string SixtyThreeBytes3 = SixtyThreeBytes + SixtyThreeBytes + SixtyThreeBytes; + + protected static IEnumerable TestCaseDefinitions { get; } = + new[] + { + //https://asecuritysite.com/encryption/xxHash, Example 1 + new TestCase( + "Nobody inspects the spammish repetition", + Encoding.ASCII.GetBytes("Nobody inspects the spammish repetition"), + "FBCEA83C8A378BF1"), + //https://asecuritysite.com/encryption/xxHash, Example 2 + new TestCase( + "The quick brown fox jumps over the lazy dog", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog"), + "0B242D361FDA71BC"), + //https://asecuritysite.com/encryption/xxHash, Example 3 + new TestCase( + "The quick brown fox jumps over the lazy dog.", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog."), + "44AD33705751AD73"), + + // Manual exploration to force boundary conditions in code coverage. + // Output values produced by existing tools. + + // The "in three pieces" test causes this to build up in the accumulator every time. + new TestCase( + "abc", + Encoding.ASCII.GetBytes("abc"), + "44BC2CF5AD770999"), + // Accumulates every time. + new TestCase( + "123456", + "313233343536", + "2B2DC38AAA53C322"), + // In the 3 pieces test, the third call to Append fills and process the holdback, + // then stores the remainder. + // The remainder is 40-32 = 8 bytes, so the Complete phase hits 1/0/0. + new TestCase( + "1234567890123456789012345678901234567890", + "31323334353637383930313233343536373839303132333435363738393031323334353637383930", + "5F3AF5E23EEB431D"), + // Same as above, but ends up with a byte that doesn't align to ulongs (1/0/1) + new TestCase( + "12345678901234567890123456789012345678901", + "3132333435363738393031323334353637383930313233343536373839303132333435363738393031", + "45A03ED59AB5CAD6"), + // Same as above, but ends up with a spare uint (1/1/0) + new TestCase( + "12345678901234567890123456789012345678901234", + "3132333435363738393031323334353637383930313233343536373839303132333435363738393031323334", + "CA5C1B5B9061279F"), + // The maximum amount of remainder work, 31 bytes (3/1/3) + new TestCase( + DotNetNCHashing, + Encoding.ASCII.GetBytes(DotNetNCHashing), + "D8444D7806DFDE0E"), + // 33 * 3 bytes long, in the "in three pieces" test each call to Append processes + // a lane and holds one byte. + new TestCase( + $"{ThirtyThreeBytes} (x3)", + Encoding.ASCII.GetBytes(ThirtyThreeBytes3), + "488DF4E623587E10"), + // 63 * 3 bytes long. In the "in three pieces" test the later calls to Append call + // into ProcessStripe with unaligned starts. + new TestCase( + $"{SixtyThreeBytes} (x3)", + Encoding.ASCII.GetBytes(SixtyThreeBytes3), + "239C7B3A85BD22B3"), + }; + + protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash64(); + + protected override byte[] StaticOneShot(byte[] source) => XxHash64.Hash(source); + + protected override byte[] StaticOneShot(ReadOnlySpan source) => XxHash64.Hash(source); + + protected override int StaticOneShot(ReadOnlySpan source, Span destination) => + XxHash64.Hash(source, destination); + + protected override bool TryStaticOneShot(ReadOnlySpan source, Span destination, out int bytesWritten) => + XxHash64.TryHash(source, destination, out bytesWritten); + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocate(TestCase testCase) + { + InstanceAppendAllocateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocateAndReset(TestCase testCase) + { + InstanceAppendAllocateAndResetDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceMultiAppendGetCurrentHash(TestCase testCase) + { + InstanceMultiAppendGetCurrentHashDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyEmptyState(TestCase testCase) + { + InstanceVerifyEmptyStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyResetState(TestCase testCase) + { + InstanceVerifyResetStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotArray(TestCase testCase) + { + StaticVerifyOneShotArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToArray(TestCase testCase) + { + StaticVerifyOneShotSpanToArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToSpan(TestCase testCase) + { + StaticVerifyOneShotSpanToSpanDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyTryOneShot(TestCase testCase) + { + StaticVerifyTryOneShotDriver(testCase); + } + } +} From 9f344391344127173b9c202ec949ddc25c3c1e2c Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Wed, 2 Jun 2021 09:56:48 -0700 Subject: [PATCH 06/18] Add support for seeded hashing to XxHashes --- .../ref/System.IO.Hashing.cs | 16 +- .../src/System/IO/Hashing/XxHash32.cs | 57 +++++- .../src/System/IO/Hashing/XxHash64.cs | 57 +++++- .../tests/System.IO.Hashing.Tests.csproj | 4 + .../tests/XxHash32Tests.007.cs | 157 ++++++++++++++++ .../tests/XxHash32Tests.f00d.cs | 157 ++++++++++++++++ .../tests/XxHash64Tests.007.cs | 167 ++++++++++++++++++ .../tests/XxHash64Tests.f00d.cs | 167 ++++++++++++++++++ 8 files changed, 758 insertions(+), 24 deletions(-) create mode 100644 src/libraries/System.IO.Hashing/tests/XxHash32Tests.007.cs create mode 100644 src/libraries/System.IO.Hashing/tests/XxHash32Tests.f00d.cs create mode 100644 src/libraries/System.IO.Hashing/tests/XxHash64Tests.007.cs create mode 100644 src/libraries/System.IO.Hashing/tests/XxHash64Tests.f00d.cs diff --git a/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs index bc0f746b83106e..2e103aeac0339f 100644 --- a/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs +++ b/src/libraries/System.IO.Hashing/ref/System.IO.Hashing.cs @@ -52,23 +52,27 @@ protected virtual void GetHashAndResetCore(System.Span destination) { } public sealed partial class XxHash32 : System.IO.Hashing.NonCryptographicHashAlgorithm { public XxHash32() : base (default(int)) { } + public XxHash32(int seed) : base (default(int)) { } public override void Append(System.ReadOnlySpan source) { } protected override void GetCurrentHashCore(System.Span destination) { } public static byte[] Hash(byte[] source) { throw null; } - public static byte[] Hash(System.ReadOnlySpan source) { throw null; } - public static int Hash(System.ReadOnlySpan source, System.Span destination) { throw null; } + public static byte[] Hash(byte[] source, int seed) { throw null; } + public static byte[] Hash(System.ReadOnlySpan source, int seed = 0) { throw null; } + public static int Hash(System.ReadOnlySpan source, System.Span destination, int seed = 0) { throw null; } public override void Reset() { } - public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } + public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten, int seed = 0) { throw null; } } public sealed partial class XxHash64 : System.IO.Hashing.NonCryptographicHashAlgorithm { public XxHash64() : base (default(int)) { } + public XxHash64(long seed) : base (default(int)) { } public override void Append(System.ReadOnlySpan source) { } protected override void GetCurrentHashCore(System.Span destination) { } public static byte[] Hash(byte[] source) { throw null; } - public static byte[] Hash(System.ReadOnlySpan source) { throw null; } - public static int Hash(System.ReadOnlySpan source, System.Span destination) { throw null; } + public static byte[] Hash(byte[] source, long seed) { throw null; } + public static byte[] Hash(System.ReadOnlySpan source, long seed = (long)0) { throw null; } + public static int Hash(System.ReadOnlySpan source, System.Span destination, long seed = (long)0) { throw null; } public override void Reset() { } - public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } + public static bool TryHash(System.ReadOnlySpan source, System.Span destination, out int bytesWritten, long seed = (long)0) { throw null; } } } diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs index 47fbba78338fcb..630ca1dcccc5f9 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs @@ -16,6 +16,7 @@ public sealed partial class XxHash32 : NonCryptographicHashAlgorithm private const int HashSize = sizeof(uint); private const int StripeSize = 4 * sizeof(uint); + private readonly uint _seed; private State _state; private byte[] _holdback; private int _length; @@ -23,15 +24,33 @@ public sealed partial class XxHash32 : NonCryptographicHashAlgorithm /// /// Initializes a new instance of the class. /// + /// + /// The XxHash32 algorithm supports an optional seed value. + /// Instances created with this constructor use the default seed, zero. + /// public XxHash32() + : this(0) + { + Reset(); + } + + /// + /// Initializes a new instance of the class with + /// a specified seed. + /// + /// + /// The hash seed value for computations from this instance. + /// + public XxHash32(int seed) : base(HashSize) { + _seed = unchecked((uint)seed); Reset(); } public override void Reset() { - _state = new State(0); + _state = new State(_seed); _length = 0; } @@ -111,15 +130,33 @@ public static byte[] Hash(byte[] source) return Hash(new ReadOnlySpan(source)); } + /// + /// Computes the XxHash32 hash of the provided data using the provided seed. + /// + /// The data to hash. + /// The seed value for this hash computation. + /// The XxHash32 hash of the provided data. + /// + /// is . + /// + public static byte[] Hash(byte[] source, int seed) + { + if (source is null) + throw new ArgumentNullException(nameof(source)); + + return Hash(new ReadOnlySpan(source), seed); + } + /// /// Computes the XxHash32 hash of the provided data. /// /// The data to hash. + /// The seed value for this hash computation. The default is zero. /// The XxHash32 hash of the provided data. - public static byte[] Hash(ReadOnlySpan source) + public static byte[] Hash(ReadOnlySpan source, int seed = 0) { byte[] ret = new byte[HashSize]; - StaticHash(source, ret); + StaticHash(source, ret, seed); return ret; } @@ -131,11 +168,12 @@ public static byte[] Hash(ReadOnlySpan source) /// /// On success, receives the number of bytes written to . /// + /// The seed value for this hash computation. The default is zero. /// /// if is long enough to receive /// the computed hash value (4 bytes); otherwise, . /// - public static bool TryHash(ReadOnlySpan source, Span destination, out int bytesWritten) + public static bool TryHash(ReadOnlySpan source, Span destination, out int bytesWritten, int seed = 0) { if (destination.Length < HashSize) { @@ -143,7 +181,7 @@ public static bool TryHash(ReadOnlySpan source, Span destination, ou return false; } - bytesWritten = StaticHash(source, destination); + bytesWritten = StaticHash(source, destination, seed); return true; } @@ -152,21 +190,22 @@ public static bool TryHash(ReadOnlySpan source, Span destination, ou /// /// The data to hash. /// The buffer that receives the computed hash value. + /// The seed value for this hash computation. The default is zero. /// /// The number of bytes written to . /// - public static int Hash(ReadOnlySpan source, Span destination) + public static int Hash(ReadOnlySpan source, Span destination, int seed = 0) { if (destination.Length < HashSize) throw new ArgumentException(SR.Argument_DestinationTooShort, nameof(destination)); - return StaticHash(source, destination); + return StaticHash(source, destination, seed); } - private static int StaticHash(ReadOnlySpan source, Span destination) + private static int StaticHash(ReadOnlySpan source, Span destination, int seed) { int totalLength = source.Length; - State state = new State(0); + State state = new State(unchecked((uint)seed)); while (source.Length > StripeSize) { diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs index 70a6623ede7dcc..93d0070dada85f 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs @@ -16,6 +16,7 @@ public sealed partial class XxHash64 : NonCryptographicHashAlgorithm private const int HashSize = sizeof(ulong); private const int StripeSize = 4 * sizeof(ulong); + private readonly ulong _seed; private State _state; private byte[] _holdback; private int _length; @@ -23,15 +24,33 @@ public sealed partial class XxHash64 : NonCryptographicHashAlgorithm /// /// Initializes a new instance of the class. /// + /// + /// The XxHash64 algorithm supports an optional seed value. + /// Instances created with this constructor use the default seed, zero. + /// public XxHash64() : base(HashSize) { Reset(); } + /// + /// Initializes a new instance of the class with + /// a specified seed. + /// + /// + /// The hash seed value for computations from this instance. + /// + public XxHash64(long seed) + : base(HashSize) + { + _seed = unchecked((ulong)seed); + Reset(); + } + public override void Reset() { - _state = new State(0); + _state = new State(_seed); _length = 0; } @@ -111,15 +130,33 @@ public static byte[] Hash(byte[] source) return Hash(new ReadOnlySpan(source)); } + /// + /// Computes the XxHash64 hash of the provided data using the provided seed. + /// + /// The data to hash. + /// The seed value for this hash computation. + /// The XxHash64 hash of the provided data. + /// + /// is . + /// + public static byte[] Hash(byte[] source, long seed) + { + if (source is null) + throw new ArgumentNullException(nameof(source)); + + return Hash(new ReadOnlySpan(source), seed); + } + /// /// Computes the XxHash64 hash of the provided data. /// /// The data to hash. + /// The seed value for this hash computation. The default is zero. /// The XxHash64 hash of the provided data. - public static byte[] Hash(ReadOnlySpan source) + public static byte[] Hash(ReadOnlySpan source, long seed = 0) { byte[] ret = new byte[HashSize]; - StaticHash(source, ret); + StaticHash(source, ret, seed); return ret; } @@ -131,11 +168,12 @@ public static byte[] Hash(ReadOnlySpan source) /// /// On success, receives the number of bytes written to . /// + /// The seed value for this hash computation. The default is zero. /// /// if is long enough to receive /// the computed hash value (4 bytes); otherwise, . /// - public static bool TryHash(ReadOnlySpan source, Span destination, out int bytesWritten) + public static bool TryHash(ReadOnlySpan source, Span destination, out int bytesWritten, long seed = 0) { if (destination.Length < HashSize) { @@ -143,7 +181,7 @@ public static bool TryHash(ReadOnlySpan source, Span destination, ou return false; } - bytesWritten = StaticHash(source, destination); + bytesWritten = StaticHash(source, destination, seed); return true; } @@ -152,21 +190,22 @@ public static bool TryHash(ReadOnlySpan source, Span destination, ou /// /// The data to hash. /// The buffer that receives the computed hash value. + /// The seed value for this hash computation. The default is zero. /// /// The number of bytes written to . /// - public static int Hash(ReadOnlySpan source, Span destination) + public static int Hash(ReadOnlySpan source, Span destination, long seed = 0) { if (destination.Length < HashSize) throw new ArgumentException(SR.Argument_DestinationTooShort, nameof(destination)); - return StaticHash(source, destination); + return StaticHash(source, destination, seed); } - private static int StaticHash(ReadOnlySpan source, Span destination) + private static int StaticHash(ReadOnlySpan source, Span destination, long seed) { int totalLength = source.Length; - State state = new State(0); + State state = new State(unchecked((ulong)seed)); while (source.Length > StripeSize) { diff --git a/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj index e60fbcfe0ea69f..4c6b5621e3bb1e 100644 --- a/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj +++ b/src/libraries/System.IO.Hashing/tests/System.IO.Hashing.Tests.csproj @@ -9,7 +9,11 @@ + + + + CommonTest\System\IO\PositionValueStream.cs diff --git a/src/libraries/System.IO.Hashing/tests/XxHash32Tests.007.cs b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.007.cs new file mode 100644 index 00000000000000..eed5aec365b3eb --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.007.cs @@ -0,0 +1,157 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public class XxHash32Tests_Seeded_007 : NonCryptoHashTestDriver + { + private int Seed = 0x007_007_00; + + private static readonly byte[] s_emptyHashValue = new byte[] { 0xC4, 0xD2, 0x6D, 0x9A }; + + public XxHash32Tests_Seeded_007() + : base(s_emptyHashValue) + { + } + + public static IEnumerable TestCases + { + get + { + object[] arr = new object[1]; + + foreach (TestCase testCase in TestCaseDefinitions) + { + arr[0] = testCase; + yield return arr; + } + } + } + + private const string DotNetHashesThis = ".NET Hashes This!"; + private const string DotNetHashesThis3 = DotNetHashesThis + DotNetHashesThis + DotNetHashesThis; + private const string DotNetNCHashing = ".NET now has non-crypto hashing"; + private const string DotNetNCHashing3 = DotNetNCHashing + DotNetNCHashing + DotNetNCHashing; + + protected static IEnumerable TestCaseDefinitions { get; } = + new[] + { + // Same inputs as the main XxHash32 tests, but with the seed applied. + new TestCase( + "Nobody inspects the spammish repetition", + Encoding.ASCII.GetBytes("Nobody inspects the spammish repetition"), + "21C69E3A"), + new TestCase( + "The quick brown fox jumps over the lazy dog", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog"), + "558D024f"), + new TestCase( + "The quick brown fox jumps over the lazy dog.", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog."), + "E7D2C0A7"), + new TestCase( + "abc", + Encoding.ASCII.GetBytes("abc"), + "D46070BB"), + new TestCase( + "123456", + "313233343536", + "4256C97B"), + new TestCase( + "12345678901234567890", + "3132333435363738393031323334353637383930", + "A57B8FEE"), + new TestCase( + "123456789012345678901", + "313233343536373839303132333435363738393031", + "6789AD70"), + new TestCase( + $"{DotNetHashesThis} (x3)", + Encoding.ASCII.GetBytes(DotNetHashesThis3), + "C5231E05"), + new TestCase( + $"{DotNetNCHashing} (x3)", + Encoding.ASCII.GetBytes(DotNetNCHashing3), + "CABC8ABD"), + }; + + protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash32(Seed); + + protected override byte[] StaticOneShot(byte[] source) => XxHash32.Hash(source, Seed); + + protected override byte[] StaticOneShot(ReadOnlySpan source) => XxHash32.Hash(source, Seed); + + protected override int StaticOneShot(ReadOnlySpan source, Span destination) => + XxHash32.Hash(source, destination, Seed); + + protected override bool TryStaticOneShot(ReadOnlySpan source, Span destination, out int bytesWritten) => + XxHash32.TryHash(source, destination, out bytesWritten, Seed); + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocate(TestCase testCase) + { + InstanceAppendAllocateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocateAndReset(TestCase testCase) + { + InstanceAppendAllocateAndResetDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceMultiAppendGetCurrentHash(TestCase testCase) + { + InstanceMultiAppendGetCurrentHashDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyEmptyState(TestCase testCase) + { + InstanceVerifyEmptyStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyResetState(TestCase testCase) + { + InstanceVerifyResetStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotArray(TestCase testCase) + { + StaticVerifyOneShotArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToArray(TestCase testCase) + { + StaticVerifyOneShotSpanToArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToSpan(TestCase testCase) + { + StaticVerifyOneShotSpanToSpanDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyTryOneShot(TestCase testCase) + { + StaticVerifyTryOneShotDriver(testCase); + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/XxHash32Tests.f00d.cs b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.f00d.cs new file mode 100644 index 00000000000000..5171e62f8ca562 --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/XxHash32Tests.f00d.cs @@ -0,0 +1,157 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public class XxHash32Tests_Seeded_f00d : NonCryptoHashTestDriver + { + private int Seed = unchecked((int)0xF00D_F00D); + + private static readonly byte[] s_emptyHashValue = new byte[] { 0x62, 0xB3, 0x2B, 0x9D }; + + public XxHash32Tests_Seeded_f00d() + : base(s_emptyHashValue) + { + } + + public static IEnumerable TestCases + { + get + { + object[] arr = new object[1]; + + foreach (TestCase testCase in TestCaseDefinitions) + { + arr[0] = testCase; + yield return arr; + } + } + } + + private const string DotNetHashesThis = ".NET Hashes This!"; + private const string DotNetHashesThis3 = DotNetHashesThis + DotNetHashesThis + DotNetHashesThis; + private const string DotNetNCHashing = ".NET now has non-crypto hashing"; + private const string DotNetNCHashing3 = DotNetNCHashing + DotNetNCHashing + DotNetNCHashing; + + protected static IEnumerable TestCaseDefinitions { get; } = + new[] + { + // Same inputs as the main XxHash32 tests, but with the seed applied. + new TestCase( + "Nobody inspects the spammish repetition", + Encoding.ASCII.GetBytes("Nobody inspects the spammish repetition"), + "E8FF660B"), + new TestCase( + "The quick brown fox jumps over the lazy dog", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog"), + "C2B00BA1"), + new TestCase( + "The quick brown fox jumps over the lazy dog.", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog."), + "11AC3BD7"), + new TestCase( + "abc", + Encoding.ASCII.GetBytes("abc"), + "BC85BB95"), + new TestCase( + "123456", + "313233343536", + "F549D3A7"), + new TestCase( + "12345678901234567890", + "3132333435363738393031323334353637383930", + "E6173FEA"), + new TestCase( + "123456789012345678901", + "313233343536373839303132333435363738393031", + "5F086DF1"), + new TestCase( + $"{DotNetHashesThis} (x3)", + Encoding.ASCII.GetBytes(DotNetHashesThis3), + "893F4A6F"), + new TestCase( + $"{DotNetNCHashing} (x3)", + Encoding.ASCII.GetBytes(DotNetNCHashing3), + "5A513E6D"), + }; + + protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash32(Seed); + + protected override byte[] StaticOneShot(byte[] source) => XxHash32.Hash(source, Seed); + + protected override byte[] StaticOneShot(ReadOnlySpan source) => XxHash32.Hash(source, Seed); + + protected override int StaticOneShot(ReadOnlySpan source, Span destination) => + XxHash32.Hash(source, destination, Seed); + + protected override bool TryStaticOneShot(ReadOnlySpan source, Span destination, out int bytesWritten) => + XxHash32.TryHash(source, destination, out bytesWritten, Seed); + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocate(TestCase testCase) + { + InstanceAppendAllocateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocateAndReset(TestCase testCase) + { + InstanceAppendAllocateAndResetDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceMultiAppendGetCurrentHash(TestCase testCase) + { + InstanceMultiAppendGetCurrentHashDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyEmptyState(TestCase testCase) + { + InstanceVerifyEmptyStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyResetState(TestCase testCase) + { + InstanceVerifyResetStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotArray(TestCase testCase) + { + StaticVerifyOneShotArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToArray(TestCase testCase) + { + StaticVerifyOneShotSpanToArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToSpan(TestCase testCase) + { + StaticVerifyOneShotSpanToSpanDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyTryOneShot(TestCase testCase) + { + StaticVerifyTryOneShotDriver(testCase); + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/XxHash64Tests.007.cs b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.007.cs new file mode 100644 index 00000000000000..5c2e575377870d --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.007.cs @@ -0,0 +1,167 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public class XxHash64Tests_Seeded_007 : NonCryptoHashTestDriver + { + private long Seed = 0x007_007_007_007_007; + + private static readonly byte[] s_emptyHashValue = + new byte[] { 0x62, 0xAD, 0xCA, 0xD4, 0xEC, 0x80, 0x84, 0x0E }; + + public XxHash64Tests_Seeded_007() + : base(s_emptyHashValue) + { + } + + public static IEnumerable TestCases + { + get + { + object[] arr = new object[1]; + + foreach (TestCase testCase in TestCaseDefinitions) + { + arr[0] = testCase; + yield return arr; + } + } + } + + private const string ThirtyThreeBytes = "This string has 33 ASCII bytes..."; + private const string ThirtyThreeBytes3 = ThirtyThreeBytes + ThirtyThreeBytes + ThirtyThreeBytes; + private const string DotNetNCHashing = ".NET now has non-crypto hashing"; + private const string SixtyThreeBytes = "A sixty-three byte test input requires substantial forethought!"; + private const string SixtyThreeBytes3 = SixtyThreeBytes + SixtyThreeBytes + SixtyThreeBytes; + + protected static IEnumerable TestCaseDefinitions { get; } = + new[] + { + // Same inputs as the main XxHash64 tests, but with the seed applied. + new TestCase( + "Nobody inspects the spammish repetition", + Encoding.ASCII.GetBytes("Nobody inspects the spammish repetition"), + "C86A41E2F34280A0"), + new TestCase( + "The quick brown fox jumps over the lazy dog", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog"), + "BB05857F11B054EB"), + new TestCase( + "The quick brown fox jumps over the lazy dog.", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog."), + "618682461CB28F83"), + new TestCase( + "abc", + Encoding.ASCII.GetBytes("abc"), + "6BF4B26E3CA10C20"), + new TestCase( + "123456", + "313233343536", + "CA35E96DF53D4962"), + new TestCase( + "1234567890123456789012345678901234567890", + "31323334353637383930313233343536373839303132333435363738393031323334353637383930", + "FA3195B38205C088"), + new TestCase( + "12345678901234567890123456789012345678901", + "3132333435363738393031323334353637383930313233343536373839303132333435363738393031", + "1980150281DF51A9"), + new TestCase( + "12345678901234567890123456789012345678901234", + "3132333435363738393031323334353637383930313233343536373839303132333435363738393031323334", + "4AE47735FD53BF97"), + new TestCase( + DotNetNCHashing, + Encoding.ASCII.GetBytes(DotNetNCHashing), + "D3ECF5BFE5F49B6F"), + new TestCase( + $"{ThirtyThreeBytes} (x3)", + Encoding.ASCII.GetBytes(ThirtyThreeBytes3), + "27C38ACA51CD2684"), + new TestCase( + $"{SixtyThreeBytes} (x3)", + Encoding.ASCII.GetBytes(SixtyThreeBytes3), + "D6095B93EB10BEDA"), + }; + + protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash64(Seed); + + protected override byte[] StaticOneShot(byte[] source) => XxHash64.Hash(source, Seed); + + protected override byte[] StaticOneShot(ReadOnlySpan source) => XxHash64.Hash(source, Seed); + + protected override int StaticOneShot(ReadOnlySpan source, Span destination) => + XxHash64.Hash(source, destination, Seed); + + protected override bool TryStaticOneShot(ReadOnlySpan source, Span destination, out int bytesWritten) => + XxHash64.TryHash(source, destination, out bytesWritten, Seed); + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocate(TestCase testCase) + { + InstanceAppendAllocateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocateAndReset(TestCase testCase) + { + InstanceAppendAllocateAndResetDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceMultiAppendGetCurrentHash(TestCase testCase) + { + InstanceMultiAppendGetCurrentHashDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyEmptyState(TestCase testCase) + { + InstanceVerifyEmptyStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyResetState(TestCase testCase) + { + InstanceVerifyResetStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotArray(TestCase testCase) + { + StaticVerifyOneShotArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToArray(TestCase testCase) + { + StaticVerifyOneShotSpanToArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToSpan(TestCase testCase) + { + StaticVerifyOneShotSpanToSpanDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyTryOneShot(TestCase testCase) + { + StaticVerifyTryOneShotDriver(testCase); + } + } +} diff --git a/src/libraries/System.IO.Hashing/tests/XxHash64Tests.f00d.cs b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.f00d.cs new file mode 100644 index 00000000000000..23006a571627b3 --- /dev/null +++ b/src/libraries/System.IO.Hashing/tests/XxHash64Tests.f00d.cs @@ -0,0 +1,167 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace System.IO.Hashing.Tests +{ + public class XxHash64Tests_Seeded_f00d : NonCryptoHashTestDriver + { + private long Seed = unchecked((long)0xF00D_F00D_F00D_F00D); + + private static readonly byte[] s_emptyHashValue = + new byte[] { 0x01, 0x3F, 0x6A, 0x1B, 0xB3, 0x9C, 0x10, 0x87 }; + + public XxHash64Tests_Seeded_f00d() + : base(s_emptyHashValue) + { + } + + public static IEnumerable TestCases + { + get + { + object[] arr = new object[1]; + + foreach (TestCase testCase in TestCaseDefinitions) + { + arr[0] = testCase; + yield return arr; + } + } + } + + private const string ThirtyThreeBytes = "This string has 33 ASCII bytes..."; + private const string ThirtyThreeBytes3 = ThirtyThreeBytes + ThirtyThreeBytes + ThirtyThreeBytes; + private const string DotNetNCHashing = ".NET now has non-crypto hashing"; + private const string SixtyThreeBytes = "A sixty-three byte test input requires substantial forethought!"; + private const string SixtyThreeBytes3 = SixtyThreeBytes + SixtyThreeBytes + SixtyThreeBytes; + + protected static IEnumerable TestCaseDefinitions { get; } = + new[] + { + // Same inputs as the main XxHash64 tests, but with the seed applied. + new TestCase( + "Nobody inspects the spammish repetition", + Encoding.ASCII.GetBytes("Nobody inspects the spammish repetition"), + "76E6275980CF4E30"), + new TestCase( + "The quick brown fox jumps over the lazy dog", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog"), + "5CC25b2B8248DF76"), + new TestCase( + "The quick brown fox jumps over the lazy dog.", + Encoding.ASCII.GetBytes("The quick brown fox jumps over the lazy dog."), + "10E8D9E4DA841407"), + new TestCase( + "abc", + Encoding.ASCII.GetBytes("abc"), + "AE9DA0E407940A85"), + new TestCase( + "123456", + "313233343536", + "99E249386C1EB47D"), + new TestCase( + "1234567890123456789012345678901234567890", + "31323334353637383930313233343536373839303132333435363738393031323334353637383930", + "0DA1BD86FAAA1BC8"), + new TestCase( + "12345678901234567890123456789012345678901", + "3132333435363738393031323334353637383930313233343536373839303132333435363738393031", + "913DFB6C43C01EB3"), + new TestCase( + "12345678901234567890123456789012345678901234", + "3132333435363738393031323334353637383930313233343536373839303132333435363738393031323334", + "F15334D3BCFC5841"), + new TestCase( + DotNetNCHashing, + Encoding.ASCII.GetBytes(DotNetNCHashing), + "C5551996D9F3737F"), + new TestCase( + $"{ThirtyThreeBytes} (x3)", + Encoding.ASCII.GetBytes(ThirtyThreeBytes3), + "6BD086C0CC425D9F"), + new TestCase( + $"{SixtyThreeBytes} (x3)", + Encoding.ASCII.GetBytes(SixtyThreeBytes3), + "6F1C62EB48EA2FEC"), + }; + + protected override NonCryptographicHashAlgorithm CreateInstance() => new XxHash64(Seed); + + protected override byte[] StaticOneShot(byte[] source) => XxHash64.Hash(source, Seed); + + protected override byte[] StaticOneShot(ReadOnlySpan source) => XxHash64.Hash(source, Seed); + + protected override int StaticOneShot(ReadOnlySpan source, Span destination) => + XxHash64.Hash(source, destination, Seed); + + protected override bool TryStaticOneShot(ReadOnlySpan source, Span destination, out int bytesWritten) => + XxHash64.TryHash(source, destination, out bytesWritten, Seed); + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocate(TestCase testCase) + { + InstanceAppendAllocateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceAppendAllocateAndReset(TestCase testCase) + { + InstanceAppendAllocateAndResetDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceMultiAppendGetCurrentHash(TestCase testCase) + { + InstanceMultiAppendGetCurrentHashDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyEmptyState(TestCase testCase) + { + InstanceVerifyEmptyStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void InstanceVerifyResetState(TestCase testCase) + { + InstanceVerifyResetStateDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotArray(TestCase testCase) + { + StaticVerifyOneShotArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToArray(TestCase testCase) + { + StaticVerifyOneShotSpanToArrayDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyOneShotSpanToSpan(TestCase testCase) + { + StaticVerifyOneShotSpanToSpanDriver(testCase); + } + + [Theory] + [MemberData(nameof(TestCases))] + public void StaticVerifyTryOneShot(TestCase testCase) + { + StaticVerifyTryOneShotDriver(testCase); + } + } +} From 11ffe57fa136a75a1b8d15f52d2f69872ef6b007 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Wed, 2 Jun 2021 09:57:10 -0700 Subject: [PATCH 07/18] Add some missing override apidocs --- .../src/System/IO/Hashing/XxHash32.cs | 12 ++++++++++++ .../src/System/IO/Hashing/XxHash64.cs | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs index 630ca1dcccc5f9..b3277e6480c7e7 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs @@ -48,12 +48,20 @@ public XxHash32(int seed) Reset(); } + /// + /// Resets the hash computation to the initial state. + /// public override void Reset() { _state = new State(_seed); _length = 0; } + /// + /// Appends the contents of to the data already + /// processed for the current hash computation. + /// + /// The data to process. public override void Append(ReadOnlySpan source) { // Every time we've read 16 bytes, process the stripe. @@ -97,6 +105,10 @@ public override void Append(ReadOnlySpan source) } } + /// + /// Writes the computed hash value to + /// without modifying accumulated state. + /// protected override void GetCurrentHashCore(Span destination) { unchecked diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs index 93d0070dada85f..43c940187a5c93 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs @@ -48,12 +48,20 @@ public XxHash64(long seed) Reset(); } + /// + /// Resets the hash computation to the initial state. + /// public override void Reset() { _state = new State(_seed); _length = 0; } + /// + /// Appends the contents of to the data already + /// processed for the current hash computation. + /// + /// The data to process. public override void Append(ReadOnlySpan source) { // Every time we've read 16 bytes, process the stripe. @@ -97,6 +105,10 @@ public override void Append(ReadOnlySpan source) } } + /// + /// Writes the computed hash value to + /// without modifying accumulated state. + /// protected override void GetCurrentHashCore(Span destination) { unchecked From ba892cd776d27549d996f674479bee5462df18a0 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Wed, 2 Jun 2021 10:09:44 -0700 Subject: [PATCH 08/18] Some csproj cleanup --- .../System.IO.Hashing/src/System.IO.Hashing.csproj | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj index 6ea7333e802bd2..e91da8138e3109 100644 --- a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -2,9 +2,7 @@ true enable - netstandard2.0;net461 - true @@ -28,12 +26,7 @@ - - - - - From 372c7960234c7e15d7bce726654fa3ad2e0a145f Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Wed, 2 Jun 2021 11:18:36 -0700 Subject: [PATCH 09/18] Add more variation to the Commonly Used Types --- src/libraries/System.IO.Hashing/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.IO.Hashing/Directory.Build.props b/src/libraries/System.IO.Hashing/Directory.Build.props index b7ff7c180adf80..d9947eba1a0668 100644 --- a/src/libraries/System.IO.Hashing/Directory.Build.props +++ b/src/libraries/System.IO.Hashing/Directory.Build.props @@ -6,6 +6,6 @@ Commonly Used Types: System.IO.Hashing.Crc32 -System.IO.Hashing.Crc64 +System.IO.Hashing.XxHash32 From af9117a4f4bfcc02afeb67001900503779b11726 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Wed, 2 Jun 2021 17:01:02 -0700 Subject: [PATCH 10/18] Add missing packageIndex data --- src/libraries/pkg/baseline/packageIndex.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libraries/pkg/baseline/packageIndex.json b/src/libraries/pkg/baseline/packageIndex.json index 33389f789205c8..756a38a689fc84 100644 --- a/src/libraries/pkg/baseline/packageIndex.json +++ b/src/libraries/pkg/baseline/packageIndex.json @@ -3602,6 +3602,12 @@ "4.0.1.0": "4.3.0" } }, + "System.IO.Hashing": { + "InboxOn": {}, + "AssemblyVersionInPackageVersion": { + "6.0.0.0": "6.0.0" + } + }, "System.IO.IsolatedStorage": { "StableVersions": [ "4.0.0", From 7379fde6bd168fe3b6e4936277a4921fc73b9fbb Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Wed, 2 Jun 2021 17:19:35 -0700 Subject: [PATCH 11/18] Remove unnecessary unchecked keywords --- .../src/System/IO/Hashing/Crc32.Table.cs | 2 +- .../src/System/IO/Hashing/Crc64.Table.cs | 2 +- .../src/System/IO/Hashing/XxHash32.State.cs | 73 ++++----- .../src/System/IO/Hashing/XxHash32.cs | 23 ++- .../src/System/IO/Hashing/XxHash64.State.cs | 141 ++++++++---------- .../src/System/IO/Hashing/XxHash64.cs | 23 ++- 6 files changed, 117 insertions(+), 147 deletions(-) diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Table.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Table.cs index b8188f1f0af59b..6dc3fd7c518440 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Table.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc32.Table.cs @@ -17,7 +17,7 @@ private static uint[] GenerateReflectedTable(uint reflectedPolynomial) for (int i = 0; i < 256; i++) { - uint val = unchecked((uint)i); + uint val = (uint)i; for (int j = 0; j < 8; j++) { diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.Table.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.Table.cs index 431c0b89a6f6ad..886589e61a36c1 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.Table.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/Crc64.Table.cs @@ -14,7 +14,7 @@ private static ulong[] GenerateTable(ulong polynomial) for (int i = 0; i < 256; i++) { - ulong val = unchecked((ulong)i) << 56; + ulong val = (ulong)i << 56; for (int j = 0; j < 8; j++) { diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs index ed10b41e39e55c..da942e3f5ec60e 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs @@ -28,15 +28,12 @@ private struct State internal State(uint seed) { - unchecked - { - _acc1 = seed + Prime32_1 + Prime32_2; - _acc2 = seed + Prime32_2; - _acc3 = seed; - _acc4 = seed - Prime32_1; + _acc1 = seed + Prime32_1 + Prime32_2; + _acc2 = seed + Prime32_2; + _acc3 = seed; + _acc4 = seed - Prime32_1; - _smallAcc = seed + Prime32_5; - } + _smallAcc = seed + Prime32_5; } internal void ProcessStripe(ReadOnlySpan source) @@ -61,52 +58,46 @@ private readonly uint Converge() private static uint ApplyRound(uint acc, ReadOnlySpan lane) { - unchecked - { - acc += BinaryPrimitives.ReadUInt32LittleEndian(lane) * Prime32_2; - acc = BitOperations.RotateLeft(acc, 13); - acc *= Prime32_1; - } + acc += BinaryPrimitives.ReadUInt32LittleEndian(lane) * Prime32_2; + acc = BitOperations.RotateLeft(acc, 13); + acc *= Prime32_1; return acc; } internal readonly uint Complete(int length, ReadOnlySpan remaining) { - unchecked - { - uint acc = length >= StripeSize ? Converge() : _smallAcc; + uint acc = length >= StripeSize ? Converge() : _smallAcc; - acc += (uint)length; + acc += (uint)length; - while (remaining.Length >= sizeof(uint)) - { - uint lane = BinaryPrimitives.ReadUInt32LittleEndian(remaining); - acc += lane * Prime32_3; - acc = BitOperations.RotateLeft(acc, 17); - acc *= Prime32_4; + while (remaining.Length >= sizeof(uint)) + { + uint lane = BinaryPrimitives.ReadUInt32LittleEndian(remaining); + acc += lane * Prime32_3; + acc = BitOperations.RotateLeft(acc, 17); + acc *= Prime32_4; - remaining = remaining.Slice(sizeof(uint)); - } + remaining = remaining.Slice(sizeof(uint)); + } - while (remaining.Length > 0) - { - uint lane = remaining[0]; - acc += lane * Prime32_5; - acc = BitOperations.RotateLeft(acc, 11); - acc *= Prime32_1; + while (remaining.Length > 0) + { + uint lane = remaining[0]; + acc += lane * Prime32_5; + acc = BitOperations.RotateLeft(acc, 11); + acc *= Prime32_1; - remaining = remaining.Slice(1); - } + remaining = remaining.Slice(1); + } - acc ^= (acc >> 15); - acc *= Prime32_2; - acc ^= (acc >> 13); - acc *= Prime32_3; - acc ^= (acc >> 16); + acc ^= (acc >> 15); + acc *= Prime32_2; + acc ^= (acc >> 13); + acc *= Prime32_3; + acc ^= (acc >> 16); - return acc; - } + return acc; } } } diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs index b3277e6480c7e7..99cf666ebdcba1 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs @@ -44,7 +44,7 @@ public XxHash32() public XxHash32(int seed) : base(HashSize) { - _seed = unchecked((uint)seed); + _seed = (uint)seed; Reset(); } @@ -111,19 +111,16 @@ public override void Append(ReadOnlySpan source) /// protected override void GetCurrentHashCore(Span destination) { - unchecked - { - int remainingLength = _length & 0x0F; - ReadOnlySpan remaining = ReadOnlySpan.Empty; - - if (remainingLength > 0) - { - remaining = new ReadOnlySpan(_holdback, 0, remainingLength); - } + int remainingLength = _length & 0x0F; + ReadOnlySpan remaining = ReadOnlySpan.Empty; - uint acc = _state.Complete(_length, remaining); - BinaryPrimitives.WriteUInt32BigEndian(destination, acc); + if (remainingLength > 0) + { + remaining = new ReadOnlySpan(_holdback, 0, remainingLength); } + + uint acc = _state.Complete(_length, remaining); + BinaryPrimitives.WriteUInt32BigEndian(destination, acc); } /// @@ -217,7 +214,7 @@ public static int Hash(ReadOnlySpan source, Span destination, int se private static int StaticHash(ReadOnlySpan source, Span destination, int seed) { int totalLength = source.Length; - State state = new State(unchecked((uint)seed)); + State state = new State((uint)seed); while (source.Length > StripeSize) { diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs index 912f422cfe8cb3..6685a07d8808f6 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs @@ -27,15 +27,12 @@ private struct State internal State(ulong seed) { - unchecked - { - _acc1 = seed + Prime64_1 + Prime64_2; - _acc2 = seed + Prime64_2; - _acc3 = seed; - _acc4 = seed - Prime64_1; + _acc1 = seed + Prime64_1 + Prime64_2; + _acc2 = seed + Prime64_2; + _acc3 = seed; + _acc4 = seed - Prime64_1; - _smallAcc = seed + Prime64_5; - } + _smallAcc = seed + Prime64_5; } internal void ProcessStripe(ReadOnlySpan source) @@ -50,33 +47,27 @@ internal void ProcessStripe(ReadOnlySpan source) private static ulong MergeAccumulator(ulong acc, ulong accN) { - unchecked - { - acc ^= ApplyRound(0, accN); - acc *= Prime64_1; - acc += Prime64_4; - } + acc ^= ApplyRound(0, accN); + acc *= Prime64_1; + acc += Prime64_4; return acc; } private readonly ulong Converge() { - unchecked - { - ulong acc = - BitOperations.RotateLeft(_acc1, 1) + - BitOperations.RotateLeft(_acc2, 7) + - BitOperations.RotateLeft(_acc3, 12) + - BitOperations.RotateLeft(_acc4, 18); - - acc = MergeAccumulator(acc, _acc1); - acc = MergeAccumulator(acc, _acc2); - acc = MergeAccumulator(acc, _acc3); - acc = MergeAccumulator(acc, _acc4); - - return acc; - } + ulong acc = + BitOperations.RotateLeft(_acc1, 1) + + BitOperations.RotateLeft(_acc2, 7) + + BitOperations.RotateLeft(_acc3, 12) + + BitOperations.RotateLeft(_acc4, 18); + + acc = MergeAccumulator(acc, _acc1); + acc = MergeAccumulator(acc, _acc2); + acc = MergeAccumulator(acc, _acc3); + acc = MergeAccumulator(acc, _acc4); + + return acc; } private static ulong ApplyRound(ulong acc, ReadOnlySpan lane) @@ -86,65 +77,59 @@ private static ulong ApplyRound(ulong acc, ReadOnlySpan lane) private static ulong ApplyRound(ulong acc, ulong lane) { - unchecked - { - acc += lane * Prime64_2; - acc = BitOperations.RotateLeft(acc, 31); - acc *= Prime64_1; - } + acc += lane * Prime64_2; + acc = BitOperations.RotateLeft(acc, 31); + acc *= Prime64_1; return acc; } internal readonly ulong Complete(int length, ReadOnlySpan remaining) { - unchecked + ulong acc = length >= StripeSize ? Converge() : _smallAcc; + + acc += (ulong)length; + + while (remaining.Length >= sizeof(ulong)) + { + ulong lane = BinaryPrimitives.ReadUInt64LittleEndian(remaining); + acc ^= ApplyRound(0, lane); + acc = BitOperations.RotateLeft(acc, 27); + acc *= Prime64_1; + acc += Prime64_4; + + remaining = remaining.Slice(sizeof(ulong)); + } + + // Doesn't need to be a while since it can occur at most once. + if (remaining.Length >= sizeof(uint)) { - ulong acc = length >= StripeSize ? Converge() : _smallAcc; - - acc += (ulong)length; - - while (remaining.Length >= sizeof(ulong)) - { - ulong lane = BinaryPrimitives.ReadUInt64LittleEndian(remaining); - acc ^= ApplyRound(0, lane); - acc = BitOperations.RotateLeft(acc, 27); - acc *= Prime64_1; - acc += Prime64_4; - - remaining = remaining.Slice(sizeof(ulong)); - } - - // Doesn't need to be a while since it can occur at most once. - if (remaining.Length >= sizeof(uint)) - { - ulong lane = BinaryPrimitives.ReadUInt32LittleEndian(remaining); - acc ^= lane * Prime64_1; - acc = BitOperations.RotateLeft(acc, 23); - acc *= Prime64_2; - acc += Prime64_3; - - remaining = remaining.Slice(sizeof(uint)); - } - - while (remaining.Length > 0) - { - ulong lane = remaining[0]; - acc ^= lane * Prime64_5; - acc = BitOperations.RotateLeft(acc, 11); - acc *= Prime64_1; - - remaining = remaining.Slice(1); - } - - acc ^= (acc >> 33); + ulong lane = BinaryPrimitives.ReadUInt32LittleEndian(remaining); + acc ^= lane * Prime64_1; + acc = BitOperations.RotateLeft(acc, 23); acc *= Prime64_2; - acc ^= (acc >> 29); - acc *= Prime64_3; - acc ^= (acc >> 32); + acc += Prime64_3; - return acc; + remaining = remaining.Slice(sizeof(uint)); } + + while (remaining.Length > 0) + { + ulong lane = remaining[0]; + acc ^= lane * Prime64_5; + acc = BitOperations.RotateLeft(acc, 11); + acc *= Prime64_1; + + remaining = remaining.Slice(1); + } + + acc ^= (acc >> 33); + acc *= Prime64_2; + acc ^= (acc >> 29); + acc *= Prime64_3; + acc ^= (acc >> 32); + + return acc; } } } diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs index 43c940187a5c93..013d1581710ad8 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs @@ -44,7 +44,7 @@ public XxHash64() public XxHash64(long seed) : base(HashSize) { - _seed = unchecked((ulong)seed); + _seed = (ulong)seed; Reset(); } @@ -111,19 +111,16 @@ public override void Append(ReadOnlySpan source) /// protected override void GetCurrentHashCore(Span destination) { - unchecked - { - int remainingLength = _length & 0x1F; - ReadOnlySpan remaining = ReadOnlySpan.Empty; - - if (remainingLength > 0) - { - remaining = new ReadOnlySpan(_holdback, 0, remainingLength); - } + int remainingLength = _length & 0x1F; + ReadOnlySpan remaining = ReadOnlySpan.Empty; - ulong acc = _state.Complete(_length, remaining); - BinaryPrimitives.WriteUInt64BigEndian(destination, acc); + if (remainingLength > 0) + { + remaining = new ReadOnlySpan(_holdback, 0, remainingLength); } + + ulong acc = _state.Complete(_length, remaining); + BinaryPrimitives.WriteUInt64BigEndian(destination, acc); } /// @@ -217,7 +214,7 @@ public static int Hash(ReadOnlySpan source, Span destination, long s private static int StaticHash(ReadOnlySpan source, Span destination, long seed) { int totalLength = source.Length; - State state = new State(unchecked((ulong)seed)); + State state = new State((ulong)seed); while (source.Length > StripeSize) { From f28576eddb716a17901a8fdd4206ad27ddc85519 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Wed, 2 Jun 2021 17:20:46 -0700 Subject: [PATCH 12/18] Improve some const folding --- .../System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs | 2 +- .../System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs index da942e3f5ec60e..fb5c6f465fb135 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs @@ -28,7 +28,7 @@ private struct State internal State(uint seed) { - _acc1 = seed + Prime32_1 + Prime32_2; + _acc1 = seed + unchecked(Prime32_1 + Prime32_2); _acc2 = seed + Prime32_2; _acc3 = seed; _acc4 = seed - Prime32_1; diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs index 6685a07d8808f6..e4c8fc0a0f4a86 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs @@ -27,7 +27,7 @@ private struct State internal State(ulong seed) { - _acc1 = seed + Prime64_1 + Prime64_2; + _acc1 = seed + unchecked(Prime64_1 + Prime64_2); _acc2 = seed + Prime64_2; _acc3 = seed; _acc4 = seed - Prime64_1; From 52275828620a5c63f4aa6bfbc015595c76f35453 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Wed, 2 Jun 2021 17:22:01 -0700 Subject: [PATCH 13/18] Make the XxHash64 ctors cascade as expected --- .../System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs index 013d1581710ad8..6c24f89ea13967 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs @@ -29,7 +29,7 @@ public sealed partial class XxHash64 : NonCryptographicHashAlgorithm /// Instances created with this constructor use the default seed, zero. /// public XxHash64() - : base(HashSize) + : this(0) { Reset(); } From cbef58517d0412ab7767560d3396ccbe6281aab9 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Tue, 8 Jun 2021 12:19:43 -0700 Subject: [PATCH 14/18] Respond to feedback --- .../src/System/IO/Hashing/XxHash32.State.cs | 7 +++---- .../System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs | 1 - .../src/System/IO/Hashing/XxHash64.State.cs | 7 +++---- .../System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs | 1 - 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs index fb5c6f465fb135..2f4dd67aa8ca24 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs @@ -39,6 +39,7 @@ internal State(uint seed) internal void ProcessStripe(ReadOnlySpan source) { Debug.Assert(source.Length >= StripeSize); + source = source.Slice(0, StripeSize); _acc1 = ApplyRound(_acc1, source); _acc2 = ApplyRound(_acc2, source.Slice(sizeof(uint))); @@ -81,14 +82,12 @@ internal readonly uint Complete(int length, ReadOnlySpan remaining) remaining = remaining.Slice(sizeof(uint)); } - while (remaining.Length > 0) + for (int i = 0; i < remaining.Length; i++) { - uint lane = remaining[0]; + uint lane = remaining[i]; acc += lane * Prime32_5; acc = BitOperations.RotateLeft(acc, 11); acc *= Prime32_1; - - remaining = remaining.Slice(1); } acc ^= (acc >> 15); diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs index 99cf666ebdcba1..91c44b709e32d8 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs @@ -31,7 +31,6 @@ public sealed partial class XxHash32 : NonCryptographicHashAlgorithm public XxHash32() : this(0) { - Reset(); } /// diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs index e4c8fc0a0f4a86..47f7ea74ca173c 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs @@ -38,6 +38,7 @@ internal State(ulong seed) internal void ProcessStripe(ReadOnlySpan source) { Debug.Assert(source.Length >= StripeSize); + source = source.Slice(0, StripeSize); _acc1 = ApplyRound(_acc1, source); _acc2 = ApplyRound(_acc2, source.Slice(sizeof(ulong))); @@ -113,14 +114,12 @@ internal readonly ulong Complete(int length, ReadOnlySpan remaining) remaining = remaining.Slice(sizeof(uint)); } - while (remaining.Length > 0) + for (int i = 0; i < remaining.Length; i++) { - ulong lane = remaining[0]; + ulong lane = remaining[i]; acc ^= lane * Prime64_5; acc = BitOperations.RotateLeft(acc, 11); acc *= Prime64_1; - - remaining = remaining.Slice(1); } acc ^= (acc >> 33); diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs index 6c24f89ea13967..cab715512d5c08 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs @@ -31,7 +31,6 @@ public sealed partial class XxHash64 : NonCryptographicHashAlgorithm public XxHash64() : this(0) { - Reset(); } /// From 74b494fc869ceede414297566e3b812b137f1a78 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Tue, 8 Jun 2021 13:22:25 -0700 Subject: [PATCH 15/18] Fix bad logic for XxHash of 2G..4G(+15 bytes) --- .../src/System/IO/Hashing/XxHash32.State.cs | 6 +++++- .../src/System/IO/Hashing/XxHash64.State.cs | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs index 2f4dd67aa8ca24..72aa582956b642 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.State.cs @@ -25,6 +25,7 @@ private struct State private uint _acc3; private uint _acc4; private readonly uint _smallAcc; + private bool _hadFullStripe; internal State(uint seed) { @@ -34,6 +35,7 @@ internal State(uint seed) _acc4 = seed - Prime32_1; _smallAcc = seed + Prime32_5; + _hadFullStripe = false; } internal void ProcessStripe(ReadOnlySpan source) @@ -45,6 +47,8 @@ internal void ProcessStripe(ReadOnlySpan source) _acc2 = ApplyRound(_acc2, source.Slice(sizeof(uint))); _acc3 = ApplyRound(_acc3, source.Slice(2 * sizeof(uint))); _acc4 = ApplyRound(_acc4, source.Slice(3 * sizeof(uint))); + + _hadFullStripe = true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -68,7 +72,7 @@ private static uint ApplyRound(uint acc, ReadOnlySpan lane) internal readonly uint Complete(int length, ReadOnlySpan remaining) { - uint acc = length >= StripeSize ? Converge() : _smallAcc; + uint acc = _hadFullStripe ? Converge() : _smallAcc; acc += (uint)length; diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs index 47f7ea74ca173c..42f66a98ab3a80 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.State.cs @@ -24,6 +24,7 @@ private struct State private ulong _acc3; private ulong _acc4; private readonly ulong _smallAcc; + private bool _hadFullStripe; internal State(ulong seed) { @@ -33,6 +34,7 @@ internal State(ulong seed) _acc4 = seed - Prime64_1; _smallAcc = seed + Prime64_5; + _hadFullStripe = false; } internal void ProcessStripe(ReadOnlySpan source) @@ -44,6 +46,8 @@ internal void ProcessStripe(ReadOnlySpan source) _acc2 = ApplyRound(_acc2, source.Slice(sizeof(ulong))); _acc3 = ApplyRound(_acc3, source.Slice(2 * sizeof(ulong))); _acc4 = ApplyRound(_acc4, source.Slice(3 * sizeof(ulong))); + + _hadFullStripe = true; } private static ulong MergeAccumulator(ulong acc, ulong accN) @@ -87,7 +91,7 @@ private static ulong ApplyRound(ulong acc, ulong lane) internal readonly ulong Complete(int length, ReadOnlySpan remaining) { - ulong acc = length >= StripeSize ? Converge() : _smallAcc; + ulong acc = _hadFullStripe ? Converge() : _smallAcc; acc += (ulong)length; From 81e82efbef19bc09ba663cc471a7e7d1762e4d9b Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Tue, 8 Jun 2021 13:28:48 -0700 Subject: [PATCH 16/18] Remove pkgproj --- src/libraries/System.IO.Hashing/Directory.Build.props | 5 ----- .../System.IO.Hashing/pkg/System.IO.Hashing.pkgproj | 9 --------- .../System.IO.Hashing/src/System.IO.Hashing.csproj | 8 ++++++++ src/libraries/pkg/baseline/packageIndex.json | 6 ------ 4 files changed, 8 insertions(+), 20 deletions(-) delete mode 100644 src/libraries/System.IO.Hashing/pkg/System.IO.Hashing.pkgproj diff --git a/src/libraries/System.IO.Hashing/Directory.Build.props b/src/libraries/System.IO.Hashing/Directory.Build.props index d9947eba1a0668..ba1f965d83cae7 100644 --- a/src/libraries/System.IO.Hashing/Directory.Build.props +++ b/src/libraries/System.IO.Hashing/Directory.Build.props @@ -2,10 +2,5 @@ Open - Provides non-cryptographic hash algorithms, such as CRC-32. - -Commonly Used Types: -System.IO.Hashing.Crc32 -System.IO.Hashing.XxHash32 diff --git a/src/libraries/System.IO.Hashing/pkg/System.IO.Hashing.pkgproj b/src/libraries/System.IO.Hashing/pkg/System.IO.Hashing.pkgproj deleted file mode 100644 index 0687eec5e0accf..00000000000000 --- a/src/libraries/System.IO.Hashing/pkg/System.IO.Hashing.pkgproj +++ /dev/null @@ -1,9 +0,0 @@ - - - - - net461;netcoreapp2.0;uap10.0.16299;$(AllXamarinFrameworks) - - - - diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj index e91da8138e3109..57d1c76fd64858 100644 --- a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -3,6 +3,14 @@ true enable netstandard2.0;net461 + true + + true + Provides non-cryptographic hash algorithms, such as CRC-32. + +Commonly Used Types: +System.IO.Hashing.Crc32 +System.IO.Hashing.XxHash32 diff --git a/src/libraries/pkg/baseline/packageIndex.json b/src/libraries/pkg/baseline/packageIndex.json index 756a38a689fc84..33389f789205c8 100644 --- a/src/libraries/pkg/baseline/packageIndex.json +++ b/src/libraries/pkg/baseline/packageIndex.json @@ -3602,12 +3602,6 @@ "4.0.1.0": "4.3.0" } }, - "System.IO.Hashing": { - "InboxOn": {}, - "AssemblyVersionInPackageVersion": { - "6.0.0.0": "6.0.0" - } - }, "System.IO.IsolatedStorage": { "StableVersions": [ "4.0.0", From ddd2c8359ca84452a72755a86cc20513d43cf1fd Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Tue, 8 Jun 2021 14:22:18 -0700 Subject: [PATCH 17/18] Add net6.0 compile target --- .../src/System.IO.Hashing.csproj | 24 ++++++++----------- .../Hashing/NonCryptographicHashAlgorithm.cs | 4 ++++ .../src/System/IO/Hashing/XxHash32.cs | 2 +- .../src/System/IO/Hashing/XxHash64.cs | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj index 57d1c76fd64858..357a8f4357c3b3 100644 --- a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -2,7 +2,7 @@ true enable - netstandard2.0;net461 + net6.0;netstandard2.0;net461 true true @@ -24,17 +24,13 @@ System.IO.Hashing.XxHash32 - - - - - - - - - - - - - + + + + + + + + + diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/NonCryptographicHashAlgorithm.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/NonCryptographicHashAlgorithm.cs index ba9e664cb9bd92..75e76c313bf8cd 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/NonCryptographicHashAlgorithm.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/NonCryptographicHashAlgorithm.cs @@ -146,7 +146,11 @@ private async Task AppendAsyncCore(Stream stream, CancellationToken cancellation while (true) { +#if NET5_0_OR_GREATER + int read = await stream.ReadAsync(buffer.AsMemory(), cancellationToken).ConfigureAwait(false); +#else int read = await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false); +#endif if (read == 0) { diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs index 91c44b709e32d8..d6db9bae19b2e3 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash32.cs @@ -18,7 +18,7 @@ public sealed partial class XxHash32 : NonCryptographicHashAlgorithm private readonly uint _seed; private State _state; - private byte[] _holdback; + private byte[]? _holdback; private int _length; /// diff --git a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs index cab715512d5c08..990ed77d64de27 100644 --- a/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs +++ b/src/libraries/System.IO.Hashing/src/System/IO/Hashing/XxHash64.cs @@ -18,7 +18,7 @@ public sealed partial class XxHash64 : NonCryptographicHashAlgorithm private readonly ulong _seed; private State _state; - private byte[] _holdback; + private byte[]? _holdback; private int _length; /// From 504154ebcdd85a7ee1bcacaf9d6995619a732134 Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Tue, 8 Jun 2021 15:07:03 -0700 Subject: [PATCH 18/18] Spell EnablePackageBaselineValidation correctly --- src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj index 357a8f4357c3b3..f4174ba3acba49 100644 --- a/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj +++ b/src/libraries/System.IO.Hashing/src/System.IO.Hashing.csproj @@ -5,7 +5,7 @@ net6.0;netstandard2.0;net461 true - true + false Provides non-cryptographic hash algorithms, such as CRC-32. Commonly Used Types: