-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
System.Text.Json: Add IAsyncEnumerable support (#50778)
* implement IAsyncEnumerable JsonConverter * Prototype of IAsyncEnumerable deserialize with Stream * Use a Queue + test buffersizes * Avoid 1 item lag * Add support for Serialize * Misc cleanup on test * extend DeserializeAsyncEnumerable test coverage also removes SerializeAsyncEnumerable components * Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs Co-authored-by: Stephen Toub <[email protected]> * address feedback * tweak test buffer values * Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs Co-authored-by: Stephen Toub <[email protected]> * Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs Co-authored-by: Stephen Toub <[email protected]> * address feedback * increase delayInterval in serialization tests * address feedback * address feedback * add test on exceptional IAsyncDisposable disposal * address feedback * Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ReadAsyncState.cs Co-authored-by: Layomi Akinrinade <[email protected]> * Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/WriteStack.cs Co-authored-by: Layomi Akinrinade <[email protected]> * fix build and remove dead code * address feedback * Revert unneeded JsonClassInfo.ElementType workaround * remove state allocation on async deserialization methods * remove tooling artifacts * address feedback * reset AsyncEnumeratorIsPendingCompletion field Co-authored-by: Steve Harter <[email protected]> Co-authored-by: Stephen Toub <[email protected]> Co-authored-by: Layomi Akinrinade <[email protected]>
- Loading branch information
4 people
authored
Apr 12, 2021
1 parent
1119725
commit 81d3a99
Showing
19 changed files
with
1,069 additions
and
118 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
.../System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableConverterFactory.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.Diagnostics; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using System.Text.Json.Serialization.Converters; | ||
|
|
||
| namespace System.Text.Json.Serialization | ||
| { | ||
| /// <summary> | ||
| /// Converter for streaming <see cref="IAsyncEnumerable{T}" /> values. | ||
| /// </summary> | ||
| internal sealed class IAsyncEnumerableConverterFactory : JsonConverterFactory | ||
| { | ||
| public override bool CanConvert(Type typeToConvert) => GetAsyncEnumerableInterface(typeToConvert) is not null; | ||
|
|
||
| public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) | ||
| { | ||
| Type? asyncEnumerableInterface = GetAsyncEnumerableInterface(typeToConvert); | ||
| Debug.Assert(asyncEnumerableInterface is not null, $"{typeToConvert} not supported by converter."); | ||
|
|
||
| Type elementType = asyncEnumerableInterface.GetGenericArguments()[0]; | ||
| Type converterType = typeof(IAsyncEnumerableOfTConverter<,>).MakeGenericType(typeToConvert, elementType); | ||
| return (JsonConverter)Activator.CreateInstance(converterType)!; | ||
| } | ||
|
|
||
| private static Type? GetAsyncEnumerableInterface(Type type) | ||
| => IEnumerableConverterFactoryHelpers.GetCompatibleGenericInterface(type, typeof(IAsyncEnumerable<>)); | ||
| } | ||
| } |
126 changes: 126 additions & 0 deletions
126
.../src/System/Text/Json/Serialization/Converters/Collection/IAsyncEnumerableOfTConverter.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.Diagnostics; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace System.Text.Json.Serialization.Converters | ||
| { | ||
| internal sealed class IAsyncEnumerableOfTConverter<TAsyncEnumerable, TElement> | ||
| : IEnumerableDefaultConverter<TAsyncEnumerable, TElement> | ||
| where TAsyncEnumerable : IAsyncEnumerable<TElement> | ||
| { | ||
| internal override bool OnTryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, ref ReadStack state, out TAsyncEnumerable value) | ||
| { | ||
| if (!typeToConvert.IsAssignableFrom(typeof(IAsyncEnumerable<TElement>))) | ||
| { | ||
| ThrowHelper.ThrowNotSupportedException_CannotPopulateCollection(TypeToConvert, ref reader, ref state); | ||
| } | ||
|
|
||
| return base.OnTryRead(ref reader, typeToConvert, options, ref state, out value!); | ||
| } | ||
|
|
||
| protected override void Add(in TElement value, ref ReadStack state) | ||
| { | ||
| ((BufferedAsyncEnumerable)state.Current.ReturnValue!)._buffer.Add(value); | ||
| } | ||
|
|
||
| protected override void CreateCollection(ref Utf8JsonReader reader, ref ReadStack state, JsonSerializerOptions options) | ||
| { | ||
| state.Current.ReturnValue = new BufferedAsyncEnumerable(); | ||
| } | ||
|
|
||
| internal override bool OnTryWrite(Utf8JsonWriter writer, TAsyncEnumerable value, JsonSerializerOptions options, ref WriteStack state) | ||
| { | ||
| if (!state.SupportContinuation) | ||
| { | ||
| ThrowHelper.ThrowNotSupportedException_TypeRequiresAsyncSerialization(TypeToConvert); | ||
| } | ||
|
|
||
| return base.OnTryWrite(writer, value, options, ref state); | ||
| } | ||
|
|
||
| [Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2012:Use ValueTasks correctly", Justification = "Converter needs to consume ValueTask's in a non-async context")] | ||
| protected override bool OnWriteResume(Utf8JsonWriter writer, TAsyncEnumerable value, JsonSerializerOptions options, ref WriteStack state) | ||
| { | ||
| IAsyncEnumerator<TElement> enumerator; | ||
| ValueTask<bool> moveNextTask; | ||
|
|
||
| if (state.Current.AsyncEnumerator is null) | ||
| { | ||
| enumerator = value.GetAsyncEnumerator(state.CancellationToken); | ||
| moveNextTask = enumerator.MoveNextAsync(); | ||
| // we always need to attach the enumerator to the stack | ||
| // since it will need to be disposed asynchronously. | ||
| state.Current.AsyncEnumerator = enumerator; | ||
| } | ||
| else | ||
| { | ||
| Debug.Assert(state.Current.AsyncEnumerator is IAsyncEnumerator<TElement>); | ||
| enumerator = (IAsyncEnumerator<TElement>)state.Current.AsyncEnumerator; | ||
|
|
||
| if (state.Current.AsyncEnumeratorIsPendingCompletion) | ||
| { | ||
| // converter was previously suspended due to a pending MoveNextAsync() task | ||
| Debug.Assert(state.PendingTask is Task<bool> && state.PendingTask.IsCompleted); | ||
| moveNextTask = new ValueTask<bool>((Task<bool>)state.PendingTask); | ||
| state.Current.AsyncEnumeratorIsPendingCompletion = false; | ||
| state.PendingTask = null; | ||
| } | ||
| else | ||
| { | ||
| // converter was suspended for a different reason; | ||
| // the last MoveNextAsync() call can only have completed with 'true'. | ||
| moveNextTask = new ValueTask<bool>(true); | ||
| } | ||
| } | ||
|
|
||
| JsonConverter<TElement> converter = GetElementConverter(ref state); | ||
|
|
||
| // iterate through the enumerator while elements are being returned synchronously | ||
| for (; moveNextTask.IsCompleted; moveNextTask = enumerator.MoveNextAsync()) | ||
| { | ||
| if (!moveNextTask.Result) | ||
| { | ||
| return true; | ||
| } | ||
|
|
||
| if (ShouldFlush(writer, ref state)) | ||
| { | ||
| return false; | ||
| } | ||
|
|
||
| TElement element = enumerator.Current; | ||
| if (!converter.TryWrite(writer, element, options, ref state)) | ||
| { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| // we have a pending MoveNextAsync() call; | ||
| // wrap inside a regular task so that it can be awaited multiple times; | ||
| // mark the current stackframe as pending completion. | ||
| Debug.Assert(state.PendingTask is null); | ||
| state.PendingTask = moveNextTask.AsTask(); | ||
| state.Current.AsyncEnumeratorIsPendingCompletion = true; | ||
| return false; | ||
| } | ||
|
|
||
| private sealed class BufferedAsyncEnumerable : IAsyncEnumerable<TElement> | ||
| { | ||
| public readonly List<TElement> _buffer = new(); | ||
|
|
||
| #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously | ||
| public async IAsyncEnumerator<TElement> GetAsyncEnumerator(CancellationToken _) | ||
| { | ||
| foreach (TElement element in _buffer) | ||
| { | ||
| yield return element; | ||
| } | ||
| } | ||
| #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.