Make low level client use System.text.json

docs/client-concepts/connection-pooling/exceptions/unexpected-exceptions.asciidoc

@@ -36,7 +36,7 @@ var audit = new Auditor(() => VirtualClusterWith <1>
     .ClientCalls(r => r.SucceedAlways())
     .ClientCalls(r => r.OnPort(9201).FailAlways(new Exception("boom!"))) <2>
     .StaticConnectionPool()
-    .Settings(s => s.DisablePing())
+    .Settings(s => s.DisablePing().EnableDebugMode())
 );
 
 audit = await audit.TraceCall(

docs/client-concepts/low-level/low-level-response-types.asciidoc

@@ -49,12 +49,19 @@ return @"{
 [source,csharp]
 ----
 var response = Client.LowLevel.Search<DynamicResponse>(PostData.Empty);
+AssertOnResponse(response);
 
-response.Get<string>("object.first").Should()
+// this uses System.Text.Json
+var responseLowLevel = LowLevelClient.Search<DynamicResponse>(PostData.Empty);
+AssertOnResponse(responseLowLevel);
+
+response.Get<string>("object.first")
+    .Should()
     .NotBeEmpty()
     .And.Be("value1");
 
-response.Get<string>("object._arbitrary_key_").Should()
+response.Get<string>("object._arbitrary_key_")
+    .Should()
     .NotBeEmpty()
     .And.Be("first");
 
@@ -65,11 +72,13 @@ response.Get<long?>("number").Should().Be(29);
 response.Get<long?>("number_does_not_exist").Should().Be(null);
 response.Get<long?>("number").Should().Be(29);
 
-response.Get<string>("array_of_objects.1.second").Should()
+response.Get<string>("array_of_objects.1.second")
+    .Should()
     .NotBeEmpty()
     .And.Be("value22");
 
-response.Get<string>("array_of_objects.1.complex\\.nested.x").Should()
+response.Get<string>("array_of_objects.1.complex\\.nested.x")
+    .Should()
     .NotBeEmpty()
     .And.Be("value6");
 ----
@@ -78,10 +87,11 @@ You can project into arrays using the dot notation
 
 [source,csharp]
 ----
-response.Get<string[]>("array_of_objects.first").Should()
+response.Get<string[]>("array_of_objects.first")
+    .Should()
     .NotBeEmpty()
     .And.HaveCount(2)
-    .And.BeEquivalentTo(new [] {"value11", "value21"});
+    .And.BeEquivalentTo(new[] { "value11", "value21" });
 ----
 
 You can even peek into array of arrays
@@ -90,8 +100,9 @@ You can even peek into array of arrays
 ----
 var nestedZs = response.Get<int[][]>("array_of_objects.nested.z.id")
     //.SelectMany(d=>d.Get<int[]>("id"))
-    .Should().NotBeEmpty()
+    .Should()
+    .NotBeEmpty()
     .And.HaveCount(2)
-    .And.BeEquivalentTo(new [] { new [] {1}, new []{3,2}});
+    .And.BeEquivalentTo(new[] { new[] { 1 }, new[] { 3, 2 } });
 ----
 

docs/code-standards/naming-conventions.asciidoc

@@ -178,6 +178,7 @@ var exceptions = new List<Type>
     elasticsearchNetAssembly.GetType("System.FormattableString"),
     elasticsearchNetAssembly.GetType("System.Runtime.CompilerServices.FormattableStringFactory"),
     elasticsearchNetAssembly.GetType("System.Runtime.CompilerServices.FormattableStringFactory"),
+    elasticsearchNetAssembly.GetType("System.Runtime.CompilerServices.FormattableStringFactory"),
     elasticsearchNetAssembly.GetType("Purify.Purifier"),
     elasticsearchNetAssembly.GetType("Purify.Purifier+IPurifier"),
     elasticsearchNetAssembly.GetType("Purify.Purifier+PurifierDotNet"),
@@ -194,6 +195,7 @@ var typesNotIElasticsearchNetNamespace = types
     .Where(t => !t.Namespace.StartsWith("Elasticsearch.Net.Utf8Json"))
     .Where(t => !t.Namespace.StartsWith("Elasticsearch.Net.Extensions"))
     .Where(t => !t.Namespace.StartsWith("Elasticsearch.Net.Diagnostics"))
+    .Where(t => !t.Namespace.StartsWith("System.Runtime.CompilerServices"))
     .Where(t => !t.Name.StartsWith("<"))
     .Where(t => IsValidTypeNameOrIdentifier(t.Name, true))
     .ToList();

src/Elasticsearch.Net.VirtualizedCluster/FixedPipelineFactory.cs

@@ -9,7 +9,7 @@ public class FixedPipelineFactory : IRequestPipelineFactory
 		public FixedPipelineFactory(IConnectionConfigurationValues connectionSettings, IDateTimeProvider dateTimeProvider)
 		{
 			DateTimeProvider = dateTimeProvider;
-			MemoryStreamFactory = RecyclableMemoryStreamFactory.Default;
+			MemoryStreamFactory = ConnectionConfiguration.DefaultMemoryStreamFactory;
 
 			Settings = connectionSettings;
 			Pipeline = Create(Settings, DateTimeProvider, MemoryStreamFactory, new SearchRequestParameters());

src/Elasticsearch.Net.VirtualizedCluster/MockResponses/SniffResponseBytes.cs

@@ -20,9 +20,9 @@ public static byte[] Create(IEnumerable<Node> nodes, string elasticsearchVersion
 				cluster_name = ClusterName,
 				nodes = SniffResponseNodes(nodes, elasticsearchVersion, publishAddressOverride, randomFqdn)
 			};
-			using (var ms = RecyclableMemoryStreamFactory.Default.Create())
+			using (var ms = ConnectionConfiguration.DefaultMemoryStreamFactory.Create())
 			{
-				LowLevelRequestResponseSerializer.Instance.Serialize(response, ms);
+				SystemTextJsonSerializer.Instance.Serialize(response, ms);
 				return ms.ToArray();
 			}
 		}

src/Elasticsearch.Net.VirtualizedCluster/Rules/RuleBase.cs

@@ -64,7 +64,7 @@ public TRule ReturnResponse<T>(T response)
 			where T : class
 		{
 			byte[] r;
-			using (var ms = RecyclableMemoryStreamFactory.Default.Create())
+			using (var ms = ConnectionConfiguration.DefaultMemoryStreamFactory.Create())
 			{
 				LowLevelRequestResponseSerializer.Instance.Serialize(response, ms);
 				r = ms.ToArray();

src/Elasticsearch.Net.VirtualizedCluster/VirtualClusterConnection.cs

@@ -281,7 +281,7 @@ private static byte[] CallResponse<TRule>(TRule rule)
 			if (_defaultResponseBytes != null) return _defaultResponseBytes;
 
 			var response = DefaultResponse;
-			using (var ms = RecyclableMemoryStreamFactory.Default.Create())
+			using (var ms = ConnectionConfiguration.DefaultMemoryStreamFactory.Create())
 			{
 				LowLevelRequestResponseSerializer.Instance.Serialize(response, ms);
 				_defaultResponseBytes = ms.ToArray();

src/Elasticsearch.Net/Configuration/ConnectionConfiguration.cs

@@ -1,4 +1,4 @@
-// Licensed to Elasticsearch B.V under one or more agreements.
+// Licensed to Elasticsearch B.V under one or more agreements.
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
@@ -200,8 +200,10 @@ public abstract class ConnectionConfiguration<T> : IConnectionConfigurationValue
 		private bool _sniffOnStartup;
 		private bool _throwExceptions;
 		private bool _transferEncodingChunked;
-		private IMemoryStreamFactory _memoryStreamFactory = RecyclableMemoryStreamFactory.Default;
+		private IMemoryStreamFactory _memoryStreamFactory = DefaultMemoryStreamFactory;
 		private bool _enableTcpStats;
+		//public static IMemoryStreamFactory Default { get; } = RecyclableMemoryStreamFactory.Default;
+		public static IMemoryStreamFactory DefaultMemoryStreamFactory { get; } = Elasticsearch.Net.MemoryStreamFactory.Default;
 		private bool _enableThreadPoolStats;
 
 		private string _userAgent = ConnectionConfiguration.DefaultUserAgent;
@@ -211,7 +213,7 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 		{
 			_connectionPool = connectionPool;
 			_connection = connection ?? new HttpConnection();
-			var serializer = requestResponseSerializer ?? new LowLevelRequestResponseSerializer();
+			var serializer = requestResponseSerializer ?? new SystemTextJsonSerializer();
 			UseThisRequestResponseSerializer = new DiagnosticsSerializerProxy(serializer);
 
 			_connectionLimit = ConnectionConfiguration.DefaultConnectionLimit;

src/Elasticsearch.Net/Elasticsearch.Net.csproj

@@ -23,9 +23,8 @@
     <PackageReference Include="Microsoft.CSharp" Version="4.6.0" />
     <PackageReference Include="System.Buffers" Version="4.5.0" />
     <PackageReference Include="System.Diagnostics.DiagnosticSource" Version="4.5.1" />
-
-    <PackageReference Condition="'$(TargetFramework)' != 'netstandard2.1'" Include="System.Memory" Version="4.5.0" />
-
+    <PackageReference Include="System.Text.Json" Version="4.7.1" />
+
     <PackageReference Condition="'$(TargetFramework)' == 'netstandard2.0'" Include="System.Reflection.Emit" Version="4.3.0" />
     <PackageReference Condition="'$(TargetFramework)' == 'netstandard2.0'" Include="System.Reflection.Emit.Lightweight" Version="4.3.0" />
   </ItemGroup>
@@ -41,6 +40,7 @@
     <InternalsVisibleTo Include="Elasticsearch.Net.DynamicObjectResolverAllowPrivateFalseExcludeNullTrueNameMutateSnakeCase" />
 
     <InternalsVisibleTo Include="Tests" />
+    <InternalsVisibleTo Include="Tests.Domain" />
 
   </ItemGroup>
   <ItemGroup>

src/Elasticsearch.Net/Extensions/JsonElementExtensions.cs

@@ -0,0 +1,38 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text.Json;
+
+namespace Elasticsearch.Net.Extensions
+{
+	internal static class JsonElementExtensions
+	{
+		/// <summary>
+		/// Fully consumes a json element representing a json object. Meaning it will attempt to unwrap all JsonElement values
+		/// recursively to their actual types. This should only be used in the context of <see cref="DynamicDictionary"/> which is
+		/// allowed to be slow yet convenient
+		/// </summary>
+		public static IDictionary<string, object> ToDictionary(this JsonElement e) =>
+			e.ValueKind switch
+			{
+				JsonValueKind.Object => e.EnumerateObject()
+					.Aggregate(new Dictionary<string, object>(), (dict, je) =>
+					{
+						dict.Add(je.Name, DynamicValue.ConsumeJsonElement(typeof(object), je.Value));
+						return dict;
+					}),
+				JsonValueKind.Array => e.EnumerateArray()
+					.Select((je, i) => (i, o: DynamicValue.ConsumeJsonElement(typeof(object), je)))
+					.Aggregate(new Dictionary<string, object>(), (dict, t) =>
+					{
+						dict.Add(t.i.ToString(CultureInfo.InvariantCulture), t.o);
+						return dict;
+					}),
+				_ => null
+			};
+	}
+}

src/Elasticsearch.Net/Responses/Dynamic/DynamicDictionary.cs

@@ -10,8 +10,11 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Dynamic;
+using System.Globalization;
 using System.Linq;
+using System.Text.Json;
 using System.Text.RegularExpressions;
+using Elasticsearch.Net.Extensions;
 using Elasticsearch.Net.Utf8Json;
 
 // ReSharper disable ArrangeMethodOrOperatorBody
@@ -45,7 +48,8 @@ public int Count
 		/// Creates a new instance of Dictionary{String,Object} using the keys and underlying object values of this DynamicDictionary instance's key values.
 		/// </summary>
 		/// <returns></returns>
-		public Dictionary<string, object> ToDictionary() => _backingDictionary.ToDictionary(kv => kv.Key, kv => kv.Value.Value);
+		public Dictionary<string, object> ToDictionary() =>
+			_backingDictionary.ToDictionary(kv => kv.Key, kv => kv.Value.Value is JsonElement e ? DynamicValue.ConsumeJsonElement(typeof(object), e) : kv.Value.Value);
 
 		/// <summary>
 		/// Returns an empty dynamic dictionary.
@@ -301,6 +305,14 @@ public static DynamicDictionary Create(IDictionary<string, object> values)
 
 			return instance;
 		}
+		/// <summary>
+		/// Creates a dynamic dictionary from an <see cref="JsonElement" /> instance.
+		/// </summary>
+		public static DynamicDictionary Create(JsonElement e)
+		{
+			var dict = e.ToDictionary();
+			return dict == null ? Empty : Create(dict);
+		}
 
 		/// <summary>
 		/// Provides the implementation for operations that set member values. Classes derived from the <see cref="T:System.Dynamic.DynamicObject" />