Merge pull request #2 from gllebede/master

MessagePack update
aspnet · Aug 22, 2022 · fe9fa08 · fe9fa08
2 parents 6caf299 + 3d1b75b
commit fe9fa08
Show file tree

Hide file tree

Showing 177 changed files with 11,833 additions and 4,014 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -28,11 +28,11 @@
     <AdditionalFiles Include="$(MSBuildThisFileDirectory)stylecop.json" Visible="false" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0" PrivateAssets="All" />
-    <PackageReference Include="Nerdbank.GitVersioning" Version="3.2.7-beta" PrivateAssets="all" />
-    <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="16.5.124-alpha" PrivateAssets="All" />
-    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
-    <PackageReference Include="StyleCop.Analyzers" Version="1.2.0-beta.164" PrivateAssets="all" />
+    <PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.2" PrivateAssets="All" />
+    <PackageReference Include="Nerdbank.GitVersioning" Version="3.4.244" PrivateAssets="all" />
+    <PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.0.64" PrivateAssets="All" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
+    <PackageReference Include="StyleCop.Analyzers" Version="1.2.0-beta.261" PrivateAssets="all" />
   </ItemGroup>
 
   <Target Name="PrepareReleaseNotes" BeforeTargets="GenerateNuspec" DependsOnTargets="GetBuildVersion">

diff --git a/Directory.Build.rsp b/Directory.Build.rsp
@@ -0,0 +1,16 @@
+#------------------------------------------------------------------------------
+# This file contains command-line options that MSBuild will process as part of
+# every build, unless the "/noautoresponse" switch is specified.
+#
+# MSBuild processes the options in this file first, before processing the
+# options on the command line. As a result, options on the command line can
+# override the options in this file. However, depending on the options being
+# set, the overriding can also result in conflicts.
+#
+# NOTE: The "/noautoresponse" switch cannot be specified in this file, nor in
+# any response file that is referenced by this file.
+#------------------------------------------------------------------------------
+/nr:false
+/m
+/verbosity:minimal
+/clp:Summary;ForceNoAlign
diff --git a/MessagePack.sln b/MessagePack.sln
diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 [![Releases](https://img.shields.io/github/release/neuecc/MessagePack-CSharp.svg)][Releases]
 
 [![Join the chat at https://gitter.im/MessagePack-CSharp/Lobby](https://badges.gitter.im/MessagePack-CSharp/Lobby.svg)](https://gitter.im/MessagePack-CSharp/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-[![Build Status](https://dev.azure.com/ils0086/MessagePack-CSharp/_apis/build/status/MessagePack-CSharp-CI)](https://dev.azure.com/ils0086/MessagePack-CSharp/_build/latest?definitionId=2)
+[![Build Status](https://dev.azure.com/ils0086/MessagePack-CSharp/_apis/build/status/MessagePack-CSharp-CI?branchName=master)](https://dev.azure.com/ils0086/MessagePack-CSharp/_build/latest?definitionId=2&branchName=master)
 
 The extremely fast [MessagePack](http://msgpack.org/) serializer for C#.
 It is 10x faster than [MsgPack-Cli](https://github.com/msgpack/msgpack-cli) and outperforms other C# serializers. MessagePack for C# also ships with built-in support for LZ4 compression - an extremely fast compression algorithm. Performance is important, particularly in applications like games, distributed computing, microservices, or data caches.
@@ -34,6 +34,7 @@ MessagePack has a compact binary size and a full set of general purpose expressi
 - [Security](#security)
 - [Performance](#performance)
     - [Deserialization Performance for different options](#deserialization-performance-for-different-options)
+    - [String interning](#string-interning)
 - [LZ4 Compression](#lz4-compression)
     - [Attributions](#attributions)
 - [Comparison with protobuf, JSON, ZeroFormatter](#comparison-with-protobuf-json-zeroformatter)
@@ -184,7 +185,7 @@ These types can serialize by default:
 * Primitives (`int`, `string`, etc...), `Enum`s, `Nullable<>`, `Lazy<>`
 * `TimeSpan`,  `DateTime`, `DateTimeOffset`
 * `Guid`, `Uri`, `Version`, `StringBuilder`
-* `BigInteger`, `Complex`
+* `BigInteger`, `Complex`, `Half`
 * `Array[]`, `Array[,]`, `Array[,,]`, `Array[,,,]`, `ArraySegment<>`, `BitArray`
 * `KeyValuePair<,>`, `Tuple<,...>`, `ValueTuple<,...>`
 * `ArrayList`, `Hashtable`
@@ -404,6 +405,8 @@ public struct Point
 }
 ```
 
+### C# 9 `record` types
+
 C# 9.0 record with primary constructor is similar immutable object, also supports serialize/deserialize.
 
 ```csharp
@@ -412,8 +415,26 @@ C# 9.0 record with primary constructor is similar immutable object, also support
 
 // use property: to set KeyAttribute
 [MessagePackObject] public record Point([property:Key(0)] int X, [property: Key(1)] int Y);
+
+// Or use explicit properties
+[MessagePackObject]
+public record Person
+{
+    [Key(0)]
+    public string FirstName { get; init; }
+
+    [Key(1)]
+    public string LastName { get; init; }
+}
 ```
 
+### C# 9 `init` property setter limitations
+
+When using `init` property setters in _generic_ classes, [a CLR bug](https://github.com/neuecc/MessagePack-CSharp/issues/1134) prevents our most efficient code generation from invoking the property setter.
+As a result, you should avoid using `init` on property setters in generic classes when using the public-only `DynamicObjectResolver`/`StandardResolver`.
+
+When using the `DynamicObjectResolverAllowPrivate`/`StandardResolverAllowPrivate` resolver the bug does not apply and you may use `init` without restriction.
+
 ## Serialization Callback
 
 Objects implementing the `IMessagePackSerializationCallbackReceiver` interface will received `OnBeforeSerialize` and `OnAfterDeserialize` calls during serialization/deserialization.
@@ -679,7 +700,7 @@ Benchmarks comparing MessagePack For C# to other serializers were run on `Window
 * Avoid string key decoding for lookup maps (string key and use automata based name lookup with inlined IL code generation, see: [AutomataDictionary](https://github.com/neuecc/MessagePack-CSharp/blob/bcedbce3fd98cb294210d6b4a22bdc4c75ccd916/src/MessagePack/Internal/AutomataDictionary.cs)
 * To encode string keys, use pre-generated member name bytes and fixed sized byte array copies in IL, see: [UnsafeMemory.cs](https://github.com/neuecc/MessagePack-CSharp/blob/f17ddc5d107d3a2f66f60398b214ef87919ff892/src/MessagePack/Internal/UnsafeMemory.cs)
 
-Before creating this library, I implemented a fast fast serializer with [ZeroFormatter#Performance](https://github.com/neuecc/ZeroFormatter#performance). This is a further evolved implementation. MessagePack for C# is always fast and optimized for all types (primitive, small struct, large object, any collections).
+Before creating this library, I implemented a fast serializer with [ZeroFormatter#Performance](https://github.com/neuecc/ZeroFormatter#performance). This is a further evolved implementation. MessagePack for C# is always fast and optimized for all types (primitive, small struct, large object, any collections).
 
 ### <a name="deserialize-performance"></a>Deserialization Performance for different options
 
@@ -738,6 +759,49 @@ Extra note, this is serialization benchmark result.
 
  Of course, `IntKey` is fastest but `StringKey` also performs reasonably well.
 
+### <a name="string-interning"></a>String interning
+
+The msgpack format does not provide for reusing strings in the data stream.
+This naturally leads the deserializer to create a new `string` object for every string encountered,
+even if it is equal to another string previously encountered.
+
+When deserializing data that may contain the same strings repeatedly it can be worthwhile
+to have the deserializer take a little extra time to check whether it has seen a given string before
+and reuse it if it has.
+
+To enable string interning on *all* string values, use a resolver that specifies `StringInterningFormatter`
+before any of the standard ones, like this:
+
+```cs
+var options = MessagePackSerializerOptions.Standard.WithResolver(
+    CompositeResolver.Create(
+        new IMessagePackFormatter[] { new StringInterningFormatter() },
+        new IFormatterResolver[] { StandardResolver.Instance }));
+
+MessagePackSerializer.Deserialize<ClassOfStrings>(data, options);
+```
+
+If you know which fields of a particular type are likely to contain duplicate strings,
+you can apply the string interning formatter to just those fields so the deserializer only pays
+for the interned string check where it matters most.
+Note that this technique requires a `[MessagePackObject]` or `[DataContract]` class.
+
+```cs
+[MessagePackObject]
+public class ClassOfStrings
+{
+    [Key(0)]
+    [MessagePackFormatter(typeof(StringInterningFormatter))]
+    public string InternedString { get; set; }
+
+    [Key(1)]
+    public string OrdinaryString { get; set; }
+}
+```
+
+If you are writing your own formatter for some type that contains strings,
+you can call on the `StringInterningFormatter` directly from your formatter as well for the strings.
+
 ## LZ4 Compression
 
 MessagePack is a fast and *compact* format but it is not compression. [LZ4](https://github.com/lz4/lz4) is an extremely fast compression algorithm, and using it MessagePack for C# can achieve extremely fast performance as well as extremely compact binary sizes!
@@ -1068,7 +1132,7 @@ Here is an example of such a custom formatter implementation. Note its use of th
 
 ```csharp
 /// <summary>Serializes a <see cref="FileInfo" /> by its full path as a string.</summary>
-public class FileInfoFormatter<T> : IMessagePackFormatter<FileInfo>
+public class FileInfoFormatter : IMessagePackFormatter<FileInfo>
 {
     public void Serialize(
       ref MessagePackWriter writer, FileInfo value, MessagePackSerializerOptions options)
@@ -1111,7 +1175,7 @@ you must precede it with a map or array header. You must read the entire map/arr
 For example:
 
 ```csharp
-public class MySpecialObjectFormatter<T> : IMessagePackFormatter<MySpecialObject>
+public class MySpecialObjectFormatter : IMessagePackFormatter<MySpecialObject>
 {
     public void Serialize(
       ref MessagePackWriter writer, MySpecialObject value, MessagePackSerializerOptions options)
@@ -1149,15 +1213,18 @@ public class MySpecialObjectFormatter<T> : IMessagePackFormatter<MySpecialObject
         int count = reader.ReadArrayHeader();
         for (int i = 0; i < count; i++)
         {
-            case 0:
-                fullName = reader.ReadString();
-                break;
-            case 1:
-                age = reader.ReadInt32();
-                break;
-            default:
-                reader.Skip();
-                break;
+            switch (i)
+            {
+                case 0:
+                    fullName = reader.ReadString();
+                    break;
+                case 1:
+                    age = reader.ReadInt32();
+                    break;
+                default:
+                    reader.Skip();
+                    break;
+            }
         }
 
         reader.Depth--;
@@ -1342,12 +1409,6 @@ internal static class SampleCustomResolverGetFormatterHelper
             return formatter;
         }
 
-        // If target type is generics, use MakeGenericType.
-        if (t.IsGenericParameter && t.GetGenericTypeDefinition() == typeof(ValueTuple<,>))
-        {
-            return Activator.CreateInstance(typeof(ValueTupleFormatter<,>).MakeGenericType(t.GenericTypeArguments));
-        }
-
         // If type can not get, must return null for fallback mechanism.
         return null;
     }
@@ -1435,30 +1496,44 @@ var resolver = MessagePack.Resolvers.CompositeResolver.Create(
 
 ## Reserved Extension Types
 
-MessagePack for C# already used some MessagePack extension type codes, be careful to use same ext code.
+MessagePack for C# already used some MessagePack extension type codes, be careful to avoid using the same ext code for other purposes.
+
+Range | Reserved for
+--|--
+\[-128, -1\] | Reserved by the msgpack spec for predefined types
+\[30, 120) | Reserved for this library's use to support common types in .NET
+
+This leaves the following ranges for your use:
+
+- \[0, 30)
+- \[120, 127]
+
+Within the *reserved* ranges, this library defines or implements extensions that use these type codes:
 
 | Code | Type | Use by |
-| ---  | ---  | --- |
-| -1 | DateTime | MessagePack-spec reserved for timestamp |
-| 30 | Vector2[] | for Unity, UnsafeBlitFormatter |
-| 31 | Vector3[] | for Unity, UnsafeBlitFormatter |
-| 32 | Vector4[] | for Unity, UnsafeBlitFormatter |
-| 33 | Quaternion[] | for Unity, UnsafeBlitFormatter |
-| 34 | Color[] | for Unity, UnsafeBlitFormatter |
-| 35 | Bounds[] | for Unity, UnsafeBlitFormatter |
-| 36 | Rect[] | for Unity, UnsafeBlitFormatter |
-| 37 | Int[] | for Unity, UnsafeBlitFormatter |
-| 38 | Float[] | for Unity, UnsafeBlitFormatter |
-| 39 | Double[] | for Unity, UnsafeBlitFormatter |
-| 98 | All | MessagePackCompression.Lz4BlockArray |
-| 99 | All | MessagePackCompression.Lz4Block |
-| 100 | object | TypelessFormatter |
+| ---- | ---- | --- |
+| -1   | DateTime | MessagePack-spec reserved for timestamp |
+| 30   | Vector2[] | for Unity, UnsafeBlitFormatter |
+| 31   | Vector3[] | for Unity, UnsafeBlitFormatter |
+| 32   | Vector4[] | for Unity, UnsafeBlitFormatter |
+| 33   | Quaternion[] | for Unity, UnsafeBlitFormatter |
+| 34   | Color[] | for Unity, UnsafeBlitFormatter |
+| 35   | Bounds[] | for Unity, UnsafeBlitFormatter |
+| 36   | Rect[] | for Unity, UnsafeBlitFormatter |
+| 37   | Int[] | for Unity, UnsafeBlitFormatter |
+| 38   | Float[] | for Unity, UnsafeBlitFormatter |
+| 39   | Double[] | for Unity, UnsafeBlitFormatter |
+| 98   | All | MessagePackCompression.Lz4BlockArray |
+| 99   | All | MessagePackCompression.Lz4Block |
+| 100  | object | TypelessFormatter |
 
 ## Unity support
 
 Unity lowest supported version is `2018.3`, API Compatibility Level supports both `.NET 4.x` and `.NET Standard 2.0`.
 
-You can install the `unitypackage` from the [releases][Releases] page. If your build targets PC, you can use it as is, but if your build targets IL2CPP, you can not use `Dynamic***Resolver`, so it is required to use pre-code generation. Please see [pre-code generation section](#aot).
+You can install the `unitypackage` from the [releases][Releases] page.
+If your build targets .NET Framework 4.x and runs on mono, you can use it as is.
+But if your build targets IL2CPP, you can not use `Dynamic***Resolver`, so it is required to use pre-code generation. Please see [pre-code generation section](#aot).
 
 MessagePack for C# includes some additional `System.*.dll` libraries that originally provides in NuGet. They are located under `Plugins`. If other packages use these libraries (e.g. Unity Collections package using `System.Runtime.CompilerServices.Unsafe.dll`), to avoid conflicts, please delete the DLL under `Plugins`.
 
@@ -1504,7 +1579,8 @@ If you want to share a class between Unity and a server, you can use `SharedProj
 By default, MessagePack for C# serializes custom objects by [generating IL](https://msdn.microsoft.com/en-us/library/system.reflection.emit.ilgenerator.aspx) on the fly at runtime to create custom, highly tuned formatters for each type. This code generation has a minor upfront performance cost.
 Because strict-AOT environments such as Xamarin and Unity IL2CPP forbid runtime code generation, MessagePack provides a way for you to run a code generator ahead of time as well.
 
-> Note: When Unity targets the PC it allows dynamic code generation, so AOT is not required.
+> Note: When using Unity, dynamic code generation only works when targeting .NET Framework 4.x + mono runtime.
+For all other Unity targets, AOT is required.
 
 If you want to avoid the upfront dynamic generation cost or you need to run on Xamarin or Unity, you need AOT code generation. `mpc` (MessagePackCompiler) is the code generator of MessagePack for C#. mpc uses [Roslyn](https://github.com/dotnet/roslyn) to analyze source code.
 
@@ -1526,7 +1602,7 @@ Check in your `.config\dotnet-tools.json` file. On another machine you can "rest
 Once you have the tool installed, simply invoke using `dotnet mpc` within your repo:
 
 ```
-dotnet mpc -h
+dotnet mpc --help
 ```
 
 Alternatively, you can download mpc from the [releases][Releases] page, that includes platform native binaries (that don't require a separate dotnet runtime).
@@ -1535,7 +1611,7 @@ Alternatively, you can download mpc from the [releases][Releases] page, that inc
 Usage: mpc [options...]
 
 Options:
-  -i, -input <String>                                Input path of analyze csproj or directory, if input multiple csproj split with ','. (Required)
+  -i, -input <String>                                Input path to MSBuild project file or the directory containing Unity source files. (Required)
   -o, -output <String>                               Output file path(.cs) or directory(multiple generate file). (Required)
   -c, -conditionalSymbol <String>                    Conditional compiler symbols, split with ','. (Default: null)
   -r, -resolverName <String>                         Set resolver name. (Default: GeneratedResolver)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -19,7 +19,7 @@ stages:
   jobs:
   - job: Windows
     pool:
-      vmImage: windows-2019
+      vmImage: windows-2022
     steps:
     - checkout: self
       clean: true
@@ -33,7 +33,9 @@ stages:
   - job: Unity
     pool:
       name: CustomAgents
-      demands: UNITYHUB_EDITORS_FOLDER_LOCATION
+      demands:
+      - UNITYHUB_EDITORS_FOLDER_LOCATION
+      - UNITYVERSION -equals 2019.1
     steps:
     - checkout: self
       clean: true
@@ -42,7 +44,7 @@ stages:
 
   - job: Linux
     pool:
-      vmImage: Ubuntu 16.04
+      vmImage: ubuntu-20.04
     steps:
     - checkout: self
       clean: true
@@ -51,7 +53,7 @@ stages:
 
   - job: macOS
     pool:
-      vmImage: macOS-10.15
+      vmImage: macOS-11
     steps:
     - checkout: self
       clean: true
@@ -62,7 +64,7 @@ stages:
 # It also helps exercise mpc so bugs don't go unnoticed.
   - job: codegen_diff
     pool:
-      vmImage: ubuntu-latest
+      vmImage: ubuntu-20.04
     steps:
     - checkout: self
       clean: true
@@ -82,7 +84,7 @@ stages:
   jobs:
   - job: push
     pool:
-      vmImage: ubuntu-latest
+      vmImage: ubuntu-20.04
     steps:
     - download: current
       artifact: nuget

diff --git a/azure-pipelines/Get-TempToolsPath.ps1 b/azure-pipelines/Get-TempToolsPath.ps1
@@ -1,7 +1,7 @@
-if ($env:AGENT_TOOLSDIRECTORY) {
-    $path = "$env:AGENT_TOOLSDIRECTORY\vs-platform\tools"
+if ($env:AGENT_TEMPDIRECTORY) {
+    $path = "$env:AGENT_TEMPDIRECTORY\$env:BUILD_BUILDID"
 } elseif ($env:localappdata) {
-    $path = "$env:localappdata\vs-platform\tools"
+    $path = "$env:localappdata\gitrepos\tools"
 } else {
     $path = "$PSScriptRoot\..\obj\tools"
 }

diff --git a/azure-pipelines/Get-nbgv.ps1 b/azure-pipelines/Get-nbgv.ps1
@@ -10,11 +10,7 @@ if ($existingTool) {
     return $existingTool.Path
 }
 
-if ($env:AGENT_TEMPDIRECTORY) {
-    $toolInstallDir = "$env:AGENT_TEMPDIRECTORY/$env:BUILD_BUILDID"
-} else {
-    $toolInstallDir = "$PSScriptRoot/../obj/tools"
-}
+$toolInstallDir = & "$PSScriptRoot/Get-TempToolsPath.ps1"
 
 $toolPath = "$toolInstallDir/nbgv"
 if (!(Test-Path $toolInstallDir)) { New-Item -Path $toolInstallDir -ItemType Directory | Out-Null }

diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml
@@ -1,12 +1,8 @@
 steps:
-# Use VSBuild to pack because `dotnet pack` can't build VSIX projects.
-- task: VSBuild@1
+- task: DotNetCoreCLI@2
   inputs:
-    vsVersion: 16.0
-    solution: MessagePack.sln
-    msbuildArgs: /t:build,pack /m /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/msbuild.binlog"
-    platform: $(BuildPlatform)
-    configuration: $(BuildConfiguration)
+    command: build
+    arguments: --configuration $(BuildConfiguration) /t:build,pack /m /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/msbuild.binlog"
   displayName: Build MessagePack.sln
 
 - task: DotNetCoreCLI@2