Chrwhit/merge master five

eng/code-quality-reports/src/main/resources/revapi/revapi.json

@@ -512,6 +512,16 @@
         "old": "field com.azure.core.amqp.exception.AmqpErrorContext.serialVersionUID",
         "new": "field com.azure.core.amqp.exception.AmqpErrorContext.serialVersionUID",
         "justification": "The field ErrorInfo was added to AmqpErrorContext, but no existing fields were removed or changed."
+      },
+      {
+          "code": "java.method.added",
+          "new": "method com.azure.core.util.HttpClientOptions com.azure.core.util.HttpClientOptions::setReadTimeout(java.time.Duration)",
+          "justification": "New Setter method."
+      },
+      {
+          "code": "java.method.added",
+          "new": "method com.azure.core.util.HttpClientOptions com.azure.core.util.HttpClientOptions::setResponseTimeout(java.time.Duration)",
+          "justification": "New Setter method."
       }
     ]
   }

eng/jacoco-test-coverage/pom.xml

@@ -194,7 +194,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-identity</artifactId>
-      <version>1.4.0-beta.1</version> <!-- {x-version-update;com.azure:azure-identity;current} -->
+      <version>1.3.1</version> <!-- {x-version-update;com.azure:azure-identity;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
@@ -204,7 +204,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-messaging-eventgrid</artifactId>
-      <version>4.4.0-beta.1</version> <!-- {x-version-update;com.azure:azure-messaging-eventgrid;current} -->
+      <version>4.4.0</version> <!-- {x-version-update;com.azure:azure-messaging-eventgrid;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
@@ -254,17 +254,17 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-storage-common</artifactId>
-      <version>12.12.0-beta.2</version> <!-- {x-version-update;com.azure:azure-storage-common;current} -->
+      <version>12.12.0</version> <!-- {x-version-update;com.azure:azure-storage-common;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-storage-blob</artifactId>
-      <version>12.12.0-beta.2</version> <!-- {x-version-update;com.azure:azure-storage-blob;current} -->
+      <version>12.12.0</version> <!-- {x-version-update;com.azure:azure-storage-blob;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-storage-blob-batch</artifactId>
-      <version>12.10.0-beta.2</version> <!-- {x-version-update;com.azure:azure-storage-blob-batch;current} -->
+      <version>12.10.0</version> <!-- {x-version-update;com.azure:azure-storage-blob-batch;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
@@ -274,7 +274,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-storage-blob-cryptography</artifactId>
-      <version>12.12.0-beta.2</version> <!-- {x-version-update;com.azure:azure-storage-blob-cryptography;current} -->
+      <version>12.12.0</version> <!-- {x-version-update;com.azure:azure-storage-blob-cryptography;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
@@ -284,22 +284,22 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-storage-file-share</artifactId>
-      <version>12.10.0-beta.2</version> <!-- {x-version-update;com.azure:azure-storage-file-share;current} -->
+      <version>12.10.0</version> <!-- {x-version-update;com.azure:azure-storage-file-share;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-storage-file-datalake</artifactId>
-      <version>12.6.0-beta.2</version> <!-- {x-version-update;com.azure:azure-storage-file-datalake;current} -->
+      <version>12.6.0</version> <!-- {x-version-update;com.azure:azure-storage-file-datalake;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-storage-internal-avro</artifactId>
-      <version>12.0.5-beta.1</version> <!-- {x-version-update;com.azure:azure-storage-internal-avro;current} -->
+      <version>12.0.5</version> <!-- {x-version-update;com.azure:azure-storage-internal-avro;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-storage-queue</artifactId>
-      <version>12.10.0-beta.2</version> <!-- {x-version-update;com.azure:azure-storage-queue;current} -->
+      <version>12.10.0</version> <!-- {x-version-update;com.azure:azure-storage-queue;current} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>

eng/versioning/version_client.txt

@@ -84,11 +84,11 @@ com.azure:azure-data-schemaregistry-avro;1.0.0-beta.4;1.0.0-beta.5
 com.azure:azure-data-tables;12.0.0-beta.7;12.0.0-beta.8
 com.azure:azure-digitaltwins-core;1.1.1;1.2.0-beta.1
 com.azure:azure-e2e;1.0.0-beta.1;1.0.0-beta.1
-com.azure:azure-identity;1.3.0;1.4.0-beta.1
+com.azure:azure-identity;1.3.0;1.3.1
 com.azure:azure-identity-perf;1.0.0-beta.1;1.0.0-beta.1
 com.azure:azure-iot-deviceupdate;1.0.0-beta.1;1.0.0-beta.2
 com.azure:azure-iot-modelsrepository;1.0.0-beta.1;1.0.0-beta.2
-com.azure:azure-messaging-eventgrid;4.3.0;4.4.0-beta.1
+com.azure:azure-messaging-eventgrid;4.3.0;4.4.0
 com.azure:azure-messaging-eventhubs;5.7.1;5.8.0-beta.1
 com.azure:azure-messaging-eventhubs-checkpointstore-blob;1.7.1;1.8.0-beta.1
 com.azure:azure-messaging-eventhubs-track1-perf;1.0.0-beta.1;1.0.0-beta.1
@@ -115,17 +115,17 @@ com.azure:azure-security-keyvault-secrets;4.2.8;4.3.0-beta.7
 com.azure:azure-sdk-template;1.2.1-beta.2;1.2.1-beta.16
 com.azure:azure-spring-data-cosmos;3.7.0;3.8.0-beta.1
 com.azure:azure-spring-data-cosmos-test;3.0.0-beta.1;3.0.0-beta.1
-com.azure:azure-storage-blob;12.11.1;12.12.0-beta.2
-com.azure:azure-storage-blob-batch;12.9.1;12.10.0-beta.2
+com.azure:azure-storage-blob;12.11.1;12.12.0
+com.azure:azure-storage-blob-batch;12.9.1;12.10.0
 com.azure:azure-storage-blob-changefeed;12.0.0-beta.6;12.0.0-beta.7
-com.azure:azure-storage-blob-cryptography;12.11.1;12.12.0-beta.2
+com.azure:azure-storage-blob-cryptography;12.11.1;12.12.0
 com.azure:azure-storage-blob-nio;12.0.0-beta.6;12.0.0-beta.7
-com.azure:azure-storage-common;12.11.1;12.12.0-beta.2
-com.azure:azure-storage-file-share;12.9.1;12.10.0-beta.2
-com.azure:azure-storage-file-datalake;12.5.1;12.6.0-beta.2
-com.azure:azure-storage-internal-avro;12.0.4;12.0.5-beta.1
+com.azure:azure-storage-common;12.11.1;12.12.0
+com.azure:azure-storage-file-share;12.9.1;12.10.0
+com.azure:azure-storage-file-datalake;12.5.1;12.6.0
+com.azure:azure-storage-internal-avro;12.0.4;12.0.5
 com.azure:azure-storage-perf;1.0.0-beta.1;1.0.0-beta.1
-com.azure:azure-storage-queue;12.9.1;12.10.0-beta.2
+com.azure:azure-storage-queue;12.9.1;12.10.0
 com.azure:azure-template-perf;1.0.0-beta.1;1.0.0-beta.1
 com.azure:azure-media-videoanalyzer-edge;1.0.0-beta.3;1.0.0-beta.4
 com.azure:azure-verticals-agrifood-farming;1.0.0-beta.2;1.0.0-beta.3

sdk/communication/azure-communication-common/README.md

@@ -19,7 +19,7 @@ It is intended to provide cross cutting concerns, e.g. authentication.
 <dependency>
     <groupId>com.azure</groupId>
     <artifactId>azure-communication-common</artifactId>
-    <version>1.0.1</version>
+    <version>1.0.2</version>
 </dependency>
 ```
 [//]: # ({x-version-update-end})

sdk/communication/azure-communication-identity/README.md

@@ -22,7 +22,7 @@ The identity package is used for managing users and tokens for Azure Communicati
 <dependency>
   <groupId>com.azure</groupId>
   <artifactId>azure-communication-identity</artifactId>
-  <version>1.1.0</version>
+  <version>1.1.1</version>
 </dependency>
 ```
 

sdk/communication/azure-communication-identity/pom.xml

@@ -157,6 +157,22 @@
           </rules>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.revapi</groupId>
+        <artifactId>revapi-maven-plugin</artifactId>
+        <version>0.11.2</version> <!-- {x-version-update;org.revapi:revapi-maven-plugin;external_dependency} -->
+        <configuration>
+          <analysisConfiguration>
+            <revapi.ignore>
+              <item>
+                <code>java.method.added</code>
+                <class>com.azure.core.util.HttpClientOptions</class>
+                <justification>Transitive from Core. Not our problem</justification>
+              </item>
+            </revapi.ignore>
+          </analysisConfiguration>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

sdk/communication/azure-communication-phonenumbers/README.md

@@ -22,7 +22,7 @@ Purchased phone numbers can come with many capabilities, depending on the countr
 <dependency>
   <groupId>com.azure</groupId>
   <artifactId>azure-communication-phonenumbers</artifactId>
-  <version>1.0.2</version>
+  <version>1.0.3</version>
 </dependency>
 ```
 

sdk/communication/azure-communication-phonenumbers/pom.xml

@@ -156,6 +156,27 @@
           </rules>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.revapi</groupId>
+        <artifactId>revapi-maven-plugin</artifactId>
+        <version>0.11.2</version> <!-- {x-version-update;org.revapi:revapi-maven-plugin;external_dependency} -->
+        <configuration>
+          <analysisConfiguration>
+            <revapi.ignore>
+              <item>
+                <code>java.method.added</code>
+                <class>com.azure.core.util.HttpClientOptions</class>
+                <justification>Transitive from Core. Not our problem</justification>
+              </item>
+              <item>
+                <code>java.method.added</code>
+                <class>com.azure.core.http.rest.PagedFlux</class>
+                <justification>Transitive from Core. Not our problem</justification>
+              </item>
+            </revapi.ignore>
+          </analysisConfiguration>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

sdk/communication/azure-communication-sms/README.md

@@ -21,7 +21,7 @@ Azure Communication SMS is used to send simple text messages.
 <dependency>
   <groupId>com.azure</groupId>
   <artifactId>azure-communication-sms</artifactId>
-  <version>1.0.2</version>
+  <version>1.0.3</version>
 </dependency>
 ```
 

sdk/communication/azure-communication-sms/pom.xml

@@ -141,6 +141,22 @@
           </rules>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.revapi</groupId>
+        <artifactId>revapi-maven-plugin</artifactId>
+        <version>0.11.2</version> <!-- {x-version-update;org.revapi:revapi-maven-plugin;external_dependency} -->
+        <configuration>
+          <analysisConfiguration>
+            <revapi.ignore>
+              <item>
+                <code>java.method.added</code>
+                <class>com.azure.core.util.HttpClientOptions</class>
+                <justification>Transitive from Core. Not our problem</justification>
+              </item>
+            </revapi.ignore>
+          </analysisConfiguration>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

sdk/communication/pom.xml

@@ -35,22 +35,22 @@
         <dependency>
           <groupId>com.azure</groupId>
           <artifactId>azure-communication-common</artifactId>
-          <version>1.1.0-beta.1</version> <!-- {x-version-update;com.azure:azure-communication-common;current} -->
+          <version>1.0.2</version> <!-- {x-version-update;com.azure:azure-communication-common;current} -->
         </dependency>
         <dependency>
           <groupId>com.azure</groupId>
           <artifactId>azure-communication-identity</artifactId>
-          <version>1.2.0-beta.1</version> <!-- {x-version-update;com.azure:azure-communication-identity;current} -->
+          <version>1.1.1</version> <!-- {x-version-update;com.azure:azure-communication-identity;current} -->
         </dependency>
         <dependency>
           <groupId>com.azure</groupId>
           <artifactId>azure-communication-phonenumbers</artifactId>
-          <version>1.1.0-beta.1</version> <!-- {x-version-update;com.azure:azure-communication-phonenumbers;current} -->
+          <version>1.0.3</version> <!-- {x-version-update;com.azure:azure-communication-phonenumbers;current} -->
         </dependency>
         <dependency>
           <groupId>com.azure</groupId>
           <artifactId>azure-communication-sms</artifactId>
-          <version>1.1.0-beta.1</version> <!-- {x-version-update;com.azure:azure-communication-sms;current} -->
+          <version>1.0.3</version> <!-- {x-version-update;com.azure:azure-communication-sms;current} -->
         </dependency>
       </dependencies>
 

sdk/cosmos/azure-cosmos-spark_3-1_2-12/src/main/scala/com/azure/cosmos/spark/CosmosTableSchemaInferrer.scala

@@ -37,6 +37,14 @@ private object CosmosTableSchemaInferrer
     ResourceIdAttributeName,
     AttachmentsAttributeName)
 
+  private val notNullableProperties = List(
+    IdAttributeName,
+    ETagAttributeName,
+    SelfAttributeName,
+    ResourceIdAttributeName,
+    TimestampAttributeName,
+    AttachmentsAttributeName)
+
   private[spark] def inferSchema(
                                   inferredItems: Seq[ObjectNode],
                                   includeSystemProperties: Boolean,
@@ -125,7 +133,7 @@ private object CosmosTableSchemaInferrer
               case anyType: DataType => field.getKey -> StructField(
                 field.getKey,
                 anyType,
-                nullable= !systemProperties.contains(field.getKey) && allowNullForInferredProperties)
+                nullable= !notNullableProperties.contains(field.getKey) && allowNullForInferredProperties)
             })
         .toSeq)
   }

sdk/cosmos/azure-cosmos-spark_3-1_2-12/src/test/scala/com/azure/cosmos/spark/CosmosTableSchemaInferrerSpec.scala

@@ -50,7 +50,7 @@ class CosmosTableSchemaInferrerSpec extends UnitSpec {
     schema.fields(1).name shouldBe "otherProperty"
     schema.fields(0).dataType shouldBe StringType
     schema.fields(1).dataType shouldBe StringType
-    schema.fields(0).nullable shouldBe true
+    schema.fields(0).nullable shouldBe false
     schema.fields(1).nullable shouldBe true
   }
 

sdk/cosmos/azure-cosmos-spark_3-1_2-12/src/test/scala/com/azure/cosmos/spark/SparkE2EQueryITest.scala

@@ -295,6 +295,12 @@ class SparkE2EQueryITest
     fieldNames.contains(CosmosTableSchemaInferrer.ResourceIdAttributeName) shouldBe true
     fieldNames.contains(CosmosTableSchemaInferrer.ETagAttributeName) shouldBe true
     fieldNames.contains(CosmosTableSchemaInferrer.AttachmentsAttributeName) shouldBe true
+
+    rowWithInference.schema(CosmosTableSchemaInferrer.SelfAttributeName).nullable shouldBe false
+    rowWithInference.schema(CosmosTableSchemaInferrer.TimestampAttributeName).nullable shouldBe false
+    rowWithInference.schema(CosmosTableSchemaInferrer.ResourceIdAttributeName).nullable shouldBe false
+    rowWithInference.schema(CosmosTableSchemaInferrer.ETagAttributeName).nullable shouldBe false
+    rowWithInference.schema(CosmosTableSchemaInferrer.AttachmentsAttributeName).nullable shouldBe false
   }
 
   "spark query" can "use schema inference with just timestamp" in {
@@ -438,7 +444,7 @@ class SparkE2EQueryITest
       "spark.cosmos.database" -> cosmosDatabase,
       "spark.cosmos.container" -> cosmosContainer,
       "spark.cosmos.read.inferSchema.enabled" -> "true",
-      "spark.cosmos.read.inferSchema.query" -> "select TOP 1 c.type, c.age, c.isAlive, c._ts from c",
+      "spark.cosmos.read.inferSchema.query" -> "select TOP 1 c.type, c.age, c.isAlive, c._ts, c.id from c",
       "spark.cosmos.read.partitioning.strategy" -> "Restrictive"
     )
 
@@ -455,9 +461,67 @@ class SparkE2EQueryITest
     val fieldNames = rowWithInference.schema.fields.map(field => field.name)
     fieldNames.contains(CosmosTableSchemaInferrer.SelfAttributeName) shouldBe false
     fieldNames.contains(CosmosTableSchemaInferrer.TimestampAttributeName) shouldBe true
+    fieldNames.contains(CosmosTableSchemaInferrer.IdAttributeName) shouldBe true
     fieldNames.contains(CosmosTableSchemaInferrer.ResourceIdAttributeName) shouldBe false
     fieldNames.contains(CosmosTableSchemaInferrer.ETagAttributeName) shouldBe false
     fieldNames.contains(CosmosTableSchemaInferrer.AttachmentsAttributeName) shouldBe false
+
+    rowWithInference.schema(CosmosTableSchemaInferrer.TimestampAttributeName).nullable shouldBe false
+    rowWithInference.schema(CosmosTableSchemaInferrer.IdAttributeName).nullable shouldBe false
+    rowWithInference.schema("type").nullable shouldBe true
+    rowWithInference.schema("age").nullable shouldBe true
+    rowWithInference.schema("isAlive").nullable shouldBe true
+  }
+
+  "spark query" can "when forceNullableProperties is false and rows have different schema" in {
+    val cosmosEndpoint = TestConfigurations.HOST
+    val cosmosMasterKey = TestConfigurations.MASTER_KEY
+    val samplingSize = 100
+    val expectedResults = samplingSize * 2
+    val container = cosmosClient.getDatabase(cosmosDatabase).getContainer(cosmosContainer)
+
+    // Inserting documents with slightly different schema
+    for( _ <- 1 to expectedResults) {
+      val objectNode = Utils.getSimpleObjectMapper.createObjectNode()
+      val arr = objectNode.putArray("object_array")
+      val nested = Utils.getSimpleObjectMapper.createObjectNode()
+      nested.put("A", "test")
+      nested.put("B", "test")
+      arr.add(nested)
+      objectNode.put("id", UUID.randomUUID().toString)
+      container.createItem(objectNode).block()
+    }
+
+    for( _ <- 1 to samplingSize) {
+      val objectNode2 = Utils.getSimpleObjectMapper.createObjectNode()
+      val arr = objectNode2.putArray("object_array")
+      val nested = Utils.getSimpleObjectMapper.createObjectNode()
+      nested.put("A", "test")
+      arr.add(nested)
+      objectNode2.put("id", UUID.randomUUID().toString)
+      container.createItem(objectNode2).block()
+    }
+
+    val cfgWithInference = Map("spark.cosmos.accountEndpoint" -> cosmosEndpoint,
+      "spark.cosmos.accountKey" -> cosmosMasterKey,
+      "spark.cosmos.database" -> cosmosDatabase,
+      "spark.cosmos.container" -> cosmosContainer,
+      "spark.cosmos.read.inferSchema.enabled" -> "true",
+      "spark.cosmos.read.inferSchema.forceNullableProperties" -> "false",
+      "spark.cosmos.read.inferSchema.samplingSize" -> samplingSize.toString,
+      "spark.cosmos.read.inferSchema.query" -> "SELECT * FROM c ORDER BY c._ts",
+      "spark.cosmos.read.partitioning.strategy" -> "Restrictive"
+    )
+
+    val dfWithInference = spark.read.format("cosmos.oltp").options(cfgWithInference).load()
+    try {
+      dfWithInference.collect()
+      fail("Should have thrown an exception")
+    }
+    catch {
+      case inner: Exception =>
+        inner.toString.contains("The 1th field 'B' of input row cannot be null") shouldBe true
+    }
   }
 
   "spark query" can "when forceNullableProperties is false and rows have different schema" in {