feat(common): support null value for reference type collection elemen…

…ts (#3163)

- update `SerializationBuilder` to set `ValueSerialization.IsNullable : true` for reference type collection elements
  - that will add null value handling in deserialization logic for collection elements of reference type
- generate null check in generated serialization/deserialization codes for:
  - nullable value type
  - list/dictionary types
  - some framework reference types
- optimize null value processing so that we don't generate null check codes
  - for custom model types, put null value processing logic into the deserialization method of each model
  - for string, use default serialization/deserialization methods which can handle null value
  - update `byte[]` serializatoin/deserialization methods so that they can handle null value, so that we don't generate null check codes externally
- fix 3 test cases previously do not accept null value in collections in response, now we align with other language implementaion on that

resolve #3159

samples/AnomalyDetector/Generated/Models/AlignPolicy.Serialization.cs

@@ -57,6 +57,10 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
 
         internal static AlignPolicy DeserializeAlignPolicy(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<AlignMode?> alignMode = default;
             Optional<FillNAMethod?> fillNAMethod = default;
             Optional<float?> paddingValue = default;

samples/AnomalyDetector/Generated/Models/AnomalyDetectionModel.Serialization.cs

@@ -16,6 +16,10 @@ public partial class AnomalyDetectionModel
     {
         internal static AnomalyDetectionModel DeserializeAnomalyDetectionModel(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             string modelId = default;
             DateTimeOffset createdTime = default;
             DateTimeOffset lastUpdatedTime = default;

samples/AnomalyDetector/Generated/Models/AnomalyInterpretation.Serialization.cs

@@ -15,6 +15,10 @@ public partial class AnomalyInterpretation
     {
         internal static AnomalyInterpretation DeserializeAnomalyInterpretation(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<string> variable = default;
             Optional<float?> contributionScore = default;
             Optional<CorrelationChanges> correlationChanges = default;

samples/AnomalyDetector/Generated/Models/AnomalyState.Serialization.cs

@@ -17,6 +17,10 @@ public partial class AnomalyState
     {
         internal static AnomalyState DeserializeAnomalyState(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             DateTimeOffset timestamp = default;
             Optional<AnomalyValue> value = default;
             Optional<IReadOnlyList<ErrorResponse>> errors = default;

samples/AnomalyDetector/Generated/Models/AnomalyValue.Serialization.cs

@@ -16,6 +16,10 @@ public partial class AnomalyValue
     {
         internal static AnomalyValue DeserializeAnomalyValue(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             bool isAnomaly = default;
             float severity = default;
             float score = default;

samples/AnomalyDetector/Generated/Models/CorrelationChanges.Serialization.cs

@@ -16,6 +16,10 @@ public partial class CorrelationChanges
     {
         internal static CorrelationChanges DeserializeCorrelationChanges(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<IReadOnlyList<string>> changedVariables = default;
             foreach (var property in element.EnumerateObject())
             {

samples/AnomalyDetector/Generated/Models/DiagnosticsInfo.Serialization.cs

@@ -37,6 +37,10 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
 
         internal static DiagnosticsInfo DeserializeDiagnosticsInfo(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<ModelState> modelState = default;
             Optional<IList<VariableState>> variableStates = default;
             foreach (var property in element.EnumerateObject())

samples/AnomalyDetector/Generated/Models/ErrorResponse.Serialization.cs

@@ -25,6 +25,10 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
 
         internal static ErrorResponse DeserializeErrorResponse(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             string code = default;
             string message = default;
             foreach (var property in element.EnumerateObject())

samples/AnomalyDetector/Generated/Models/ModelInfo.Serialization.cs

@@ -80,6 +80,10 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
 
         internal static ModelInfo DeserializeModelInfo(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             string dataSource = default;
             Optional<DataSchema?> dataSchema = default;
             DateTimeOffset startTime = default;

samples/AnomalyDetector/Generated/Models/ModelState.Serialization.cs

@@ -62,6 +62,10 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
 
         internal static ModelState DeserializeModelState(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<IList<int>> epochIds = default;
             Optional<IList<float>> trainLosses = default;
             Optional<IList<float>> validationLosses = default;

samples/AnomalyDetector/Generated/Models/MultivariateBatchDetectionOptions.Serialization.cs

@@ -30,6 +30,10 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
 
         internal static MultivariateBatchDetectionOptions DeserializeMultivariateBatchDetectionOptions(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             string dataSource = default;
             int topContributorCount = default;
             DateTimeOffset startTime = default;

samples/AnomalyDetector/Generated/Models/MultivariateBatchDetectionResultSummary.Serialization.cs

@@ -16,6 +16,10 @@ public partial class MultivariateBatchDetectionResultSummary
     {
         internal static MultivariateBatchDetectionResultSummary DeserializeMultivariateBatchDetectionResultSummary(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             MultivariateBatchDetectionStatus status = default;
             Optional<IReadOnlyList<ErrorResponse>> errors = default;
             Optional<IReadOnlyList<VariableState>> variableStates = default;

samples/AnomalyDetector/Generated/Models/MultivariateDetectionResult.Serialization.cs

@@ -16,6 +16,10 @@ public partial class MultivariateDetectionResult
     {
         internal static MultivariateDetectionResult DeserializeMultivariateDetectionResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             string resultId = default;
             MultivariateBatchDetectionResultSummary summary = default;
             IReadOnlyList<AnomalyState> results = default;

samples/AnomalyDetector/Generated/Models/MultivariateLastDetectionResult.Serialization.cs

@@ -16,6 +16,10 @@ public partial class MultivariateLastDetectionResult
     {
         internal static MultivariateLastDetectionResult DeserializeMultivariateLastDetectionResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<IReadOnlyList<VariableState>> variableStates = default;
             Optional<IReadOnlyList<AnomalyState>> results = default;
             foreach (var property in element.EnumerateObject())

samples/AnomalyDetector/Generated/Models/UnivariateChangePointDetectionResult.Serialization.cs

@@ -16,6 +16,10 @@ public partial class UnivariateChangePointDetectionResult
     {
         internal static UnivariateChangePointDetectionResult DeserializeUnivariateChangePointDetectionResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<int?> period = default;
             Optional<IReadOnlyList<bool>> isChangePoint = default;
             Optional<IReadOnlyList<float>> confidenceScores = default;

samples/AnomalyDetector/Generated/Models/UnivariateEntireDetectionResult.Serialization.cs

@@ -16,6 +16,10 @@ public partial class UnivariateEntireDetectionResult
     {
         internal static UnivariateEntireDetectionResult DeserializeUnivariateEntireDetectionResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             int period = default;
             IReadOnlyList<float> expectedValues = default;
             IReadOnlyList<float> upperMargins = default;

samples/AnomalyDetector/Generated/Models/UnivariateLastDetectionResult.Serialization.cs

@@ -15,6 +15,10 @@ public partial class UnivariateLastDetectionResult
     {
         internal static UnivariateLastDetectionResult DeserializeUnivariateLastDetectionResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             int period = default;
             int suggestedWindow = default;
             float expectedValue = default;

samples/AnomalyDetector/Generated/Models/VariableState.Serialization.cs

@@ -75,6 +75,10 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
 
         internal static VariableState DeserializeVariableState(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<string> variable = default;
             Optional<float?> filledNARatio = default;
             Optional<int?> effectiveCount = default;

samples/AppConfiguration/Generated/Models/Error.Serialization.cs

@@ -14,6 +14,10 @@ internal partial class Error
     {
         internal static Error DeserializeError(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<string> type = default;
             Optional<string> title = default;
             Optional<string> name = default;

samples/AppConfiguration/Generated/Models/Key.Serialization.cs

@@ -14,6 +14,10 @@ public partial class Key
     {
         internal static Key DeserializeKey(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<string> name = default;
             foreach (var property in element.EnumerateObject())
             {

samples/AppConfiguration/Generated/Models/KeyListResult.Serialization.cs

@@ -15,6 +15,10 @@ internal partial class KeyListResult
     {
         internal static KeyListResult DeserializeKeyListResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<IReadOnlyList<Key>> items = default;
             Optional<string> nextLink = default;
             foreach (var property in element.EnumerateObject())

samples/AppConfiguration/Generated/Models/KeyValue.Serialization.cs

@@ -68,6 +68,10 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
 
         internal static KeyValue DeserializeKeyValue(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<string> key = default;
             Optional<string> label = default;
             Optional<string> contentType = default;

samples/AppConfiguration/Generated/Models/KeyValueListResult.Serialization.cs

@@ -15,6 +15,10 @@ internal partial class KeyValueListResult
     {
         internal static KeyValueListResult DeserializeKeyValueListResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<IReadOnlyList<KeyValue>> items = default;
             Optional<string> nextLink = default;
             foreach (var property in element.EnumerateObject())

samples/AppConfiguration/Generated/Models/Label.Serialization.cs

@@ -14,6 +14,10 @@ public partial class Label
     {
         internal static Label DeserializeLabel(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<string> name = default;
             foreach (var property in element.EnumerateObject())
             {

samples/AppConfiguration/Generated/Models/LabelListResult.Serialization.cs

@@ -15,6 +15,10 @@ internal partial class LabelListResult
     {
         internal static LabelListResult DeserializeLabelListResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Optional<IReadOnlyList<Label>> items = default;
             Optional<string> nextLink = default;
             foreach (var property in element.EnumerateObject())

samples/Azure.AI.FormRecognizer/Generated/Models/AnalyzeOperationResult.Serialization.cs

@@ -15,6 +15,10 @@ public partial class AnalyzeOperationResult
     {
         internal static AnalyzeOperationResult DeserializeAnalyzeOperationResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             OperationStatus status = default;
             DateTimeOffset createdDateTime = default;
             DateTimeOffset lastUpdatedDateTime = default;

samples/Azure.AI.FormRecognizer/Generated/Models/AnalyzeResult.Serialization.cs

@@ -15,6 +15,10 @@ public partial class AnalyzeResult
     {
         internal static AnalyzeResult DeserializeAnalyzeResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             string version = default;
             IReadOnlyList<ReadResult> readResults = default;
             Optional<IReadOnlyList<PageResult>> pageResults = default;

samples/Azure.AI.FormRecognizer/Generated/Models/CopyAuthorizationResult.Serialization.cs

@@ -26,6 +26,10 @@ void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
 
         internal static CopyAuthorizationResult DeserializeCopyAuthorizationResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             string modelId = default;
             string accessToken = default;
             long expirationDateTimeTicks = default;

samples/Azure.AI.FormRecognizer/Generated/Models/CopyOperationResult.Serialization.cs

@@ -15,6 +15,10 @@ public partial class CopyOperationResult
     {
         internal static CopyOperationResult DeserializeCopyOperationResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             OperationStatus status = default;
             DateTimeOffset createdDateTime = default;
             DateTimeOffset lastUpdatedDateTime = default;

samples/Azure.AI.FormRecognizer/Generated/Models/CopyResult.Serialization.cs

@@ -16,6 +16,10 @@ public partial class CopyResult
     {
         internal static CopyResult DeserializeCopyResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             Guid modelId = default;
             Optional<IReadOnlyList<ErrorInformation>> errors = default;
             foreach (var property in element.EnumerateObject())

samples/Azure.AI.FormRecognizer/Generated/Models/DataTable.Serialization.cs

@@ -15,6 +15,10 @@ public partial class DataTable
     {
         internal static DataTable DeserializeDataTable(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             int rows = default;
             int columns = default;
             IReadOnlyList<DataTableCell> cells = default;

samples/Azure.AI.FormRecognizer/Generated/Models/DataTableCell.Serialization.cs

@@ -15,6 +15,10 @@ public partial class DataTableCell
     {
         internal static DataTableCell DeserializeDataTableCell(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             int rowIndex = default;
             int columnIndex = default;
             Optional<int> rowSpan = default;

samples/Azure.AI.FormRecognizer/Generated/Models/DocumentResult.Serialization.cs

@@ -15,6 +15,10 @@ public partial class DocumentResult
     {
         internal static DocumentResult DeserializeDocumentResult(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             string docType = default;
             IReadOnlyList<int> pageRange = default;
             IReadOnlyDictionary<string, FieldValue> fields = default;

samples/Azure.AI.FormRecognizer/Generated/Models/ErrorInformation.Serialization.cs

@@ -14,6 +14,10 @@ public partial class ErrorInformation
     {
         internal static ErrorInformation DeserializeErrorInformation(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             string code = default;
             string message = default;
             foreach (var property in element.EnumerateObject())

samples/Azure.AI.FormRecognizer/Generated/Models/ErrorResponse.Serialization.cs

@@ -14,6 +14,10 @@ internal partial class ErrorResponse
     {
         internal static ErrorResponse DeserializeErrorResponse(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             ErrorInformation error = default;
             foreach (var property in element.EnumerateObject())
             {

samples/Azure.AI.FormRecognizer/Generated/Models/FieldValue.Serialization.cs

@@ -16,6 +16,10 @@ public partial class FieldValue
     {
         internal static FieldValue DeserializeFieldValue(JsonElement element)
         {
+            if (element.ValueKind == JsonValueKind.Null)
+            {
+                return null;
+            }
             FieldValueType type = default;
             Optional<string> valueString = default;
             Optional<DateTimeOffset> valueDate = default;