From b2be526359bdb08617dc19527cc487a0d810f9f4 Mon Sep 17 00:00:00 2001 From: Fabian Meiswinkel Date: Sat, 13 Jul 2024 01:15:35 +0200 Subject: [PATCH] Added optional id validation to prevent documents with invalid chars in id property to be created (#41108) * Added optional id validation to prevent documents with invalid chars in id property to be created --- .../implementation/EncryptionProcessor.java | 5 + .../implementation/EncryptionUtils.java | 2 +- .../cosmos/CosmosItemIdEncodingTest.java | 223 +++++++++++------- .../InternalObjectNodeTest.java | 12 +- sdk/cosmos/azure-cosmos/CHANGELOG.md | 1 + .../azure/cosmos/implementation/Configs.java | 27 +++ .../cosmos/implementation/HttpConstants.java | 1 + .../implementation/InternalObjectNode.java | 21 +- .../implementation/JsonSerializable.java | 4 +- .../implementation/RxDocumentClientImpl.java | 10 +- .../RxDocumentServiceRequest.java | 37 ++- .../azure/cosmos/implementation/Utils.java | 33 ++- .../clienttelemetry/ClientTelemetry.java | 3 +- .../cosmos/models/ModelBridgeInternal.java | 2 +- 14 files changed, 268 insertions(+), 113 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java index cc244b4967089..07656f30264d6 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionProcessor.java @@ -260,6 +260,11 @@ public Mono encrypt(byte[] payload) { } public Mono encrypt(JsonNode itemJObj) { + + if (itemJObj != null) { + Utils.validateIdValue(itemJObj.get(Constants.PROPERTY_NAME_ID)); + } + return encryptObjectNode(itemJObj).map( encryptedObjectNode -> EncryptionUtils.serializeJsonToByteArray( CosmosItemSerializer.DEFAULT_SERIALIZER, diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java index b05ff8c44987a..971858596c823 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/encryption/implementation/EncryptionUtils.java @@ -23,7 +23,7 @@ public class EncryptionUtils { } public static byte[] serializeJsonToByteArray(CosmosItemSerializer itemSerializer, Object object) { - return toByteArray(Utils.serializeJsonToByteBuffer(itemSerializer, object, null)); + return toByteArray(Utils.serializeJsonToByteBuffer(itemSerializer, object, null, false)); } public static ObjectMapper getSimpleObjectMapper() { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosItemIdEncodingTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosItemIdEncodingTest.java index 8409a8c86811f..2d9d50ef4b4e2 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosItemIdEncodingTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosItemIdEncodingTest.java @@ -6,6 +6,7 @@ package com.azure.cosmos; +import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.models.CosmosContainerProperties; @@ -22,7 +23,6 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Factory; -import org.testng.annotations.Ignore; import org.testng.annotations.Test; import reactor.core.Exceptions; @@ -437,7 +437,6 @@ public void idWithDisallowedCharQuestionMark() { this.executeTestCase(scenario); } - @Ignore("Throws IllegalArgumentException instead of CosmosException") @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) public void idWithDisallowedCharForwardSlash() { TestScenario scenario = new TestScenario( @@ -446,21 +445,49 @@ public void idWithDisallowedCharForwardSlash() { new TestScenarioExpectations( ConnectionMode.GATEWAY.toString(), HttpConstants.StatusCodes.CREATED, - HttpConstants.StatusCodes.NOTFOUND, - HttpConstants.StatusCodes.NOTFOUND, - HttpConstants.StatusCodes.NOTFOUND), + -1 , // Non-CosmosException - in this case IllegalArgumentException + -1, + -1), new TestScenarioExpectations( "COMPUTE_GATEWAY", HttpConstants.StatusCodes.CREATED, - HttpConstants.StatusCodes.NOTFOUND, - HttpConstants.StatusCodes.NOTFOUND, - HttpConstants.StatusCodes.NOTFOUND), + -1 , // Non-CosmosException - in this case IllegalArgumentException + -1, + -1), new TestScenarioExpectations( ConnectionMode.DIRECT.toString(), HttpConstants.StatusCodes.CREATED, - HttpConstants.StatusCodes.NOTFOUND, - HttpConstants.StatusCodes.NOTFOUND, - HttpConstants.StatusCodes.NOTFOUND)); + -1 , // Non-CosmosException - in this case IllegalArgumentException + -1, + -1)); + + this.executeTestCase(scenario); + } + + @Test(groups = { "fast", "emulator" }, timeOut = TIMEOUT) + public void idWithDisallowedCharForwardSlashButIdValidationEnabled() { + TestScenario scenario = new TestScenario( + "IdWithDisallowedCharForwardSlashButIdValidationEnabled", + "Disallowed/Chars" + UUID.randomUUID(), + true, + new TestScenarioExpectations( + ConnectionMode.GATEWAY.toString(), + HttpConstants.StatusCodes.BADREQUEST, + -1 , // Non-CosmosException - in this case IllegalArgumentException + -1, + -1), + new TestScenarioExpectations( + "COMPUTE_GATEWAY", + HttpConstants.StatusCodes.BADREQUEST, + -1 , // Non-CosmosException - in this case IllegalArgumentException + -1, + -1), + new TestScenarioExpectations( + ConnectionMode.DIRECT.toString(), + HttpConstants.StatusCodes.BADREQUEST, + -1 , // Non-CosmosException - in this case IllegalArgumentException + -1, + -1)); this.executeTestCase(scenario); } @@ -608,90 +635,102 @@ private void executeTestCase(TestScenario scenario) { logger.info("Scenario: {}, Id: \"{}\"", scenario.name, scenario.id); - try { - CosmosItemResponse response = this.container.createItem( - getDocumentDefinition(scenario.id), - new PartitionKey(scenario.id), - null); - - deserializeAndValidatePayload(response, scenario.id, expected.ExpectedCreateStatusCode); - } catch (Throwable throwable) { - CosmosException cosmosError = Utils.as(Exceptions.unwrap(throwable), CosmosException.class); - if (cosmosError == null) { - Fail.fail( - "Unexpected exception type " + Exceptions.unwrap(throwable).getClass().getName(), - throwable); - } - - logger.error(cosmosError.toString()); - - assertThat(cosmosError.getStatusCode()) - .isEqualTo(expected.ExpectedCreateStatusCode); - - return; + if (scenario.idValidationEnabled) { + System.setProperty(Configs.PREVENT_INVALID_ID_CHARS, "false"); } try { - CosmosItemResponse response = this.container.readItem( - scenario.id, - new PartitionKey(scenario.id), - ObjectNode.class); - - deserializeAndValidatePayload(response, scenario.id, expected.ExpectedReadStatusCode); - } catch (Throwable throwable) { - CosmosException cosmosError = Utils.as(Exceptions.unwrap(throwable), CosmosException.class); - if (cosmosError == null) { - Fail.fail( - "Unexpected exception type " + Exceptions.unwrap(throwable).getClass().getName(), - throwable); - } - if (cosmosError.getStatusCode() == 0 && - cosmosError.getCause() instanceof IllegalArgumentException && - cosmosError.getCause().getCause() instanceof JsonParseException && - cosmosError.getCause().getCause().toString().contains("Bad Request")) { + try { + CosmosItemResponse response = this.container.createItem( + getDocumentDefinition(scenario.id), + new PartitionKey(scenario.id), + null); + + deserializeAndValidatePayload(response, scenario.id, expected.ExpectedCreateStatusCode); + } catch (Throwable throwable) { + CosmosException cosmosError = Utils.as(Exceptions.unwrap(throwable), CosmosException.class); + if (cosmosError == null) { + Fail.fail( + "Unexpected exception type " + Exceptions.unwrap(throwable).getClass().getName(), + throwable); + } + + logger.error(cosmosError.toString()); + + assertThat(cosmosError.getStatusCode()) + .isEqualTo(expected.ExpectedCreateStatusCode); - logger.info("HTML BAD REQUEST", cosmosError); - assertThat(expected.ExpectedReadStatusCode).isEqualTo(400); return; - } else { - logger.info("BAD REQUEST", cosmosError); - assertThat(cosmosError.getStatusCode()).isEqualTo(expected.ExpectedReadStatusCode); } - } - try { - CosmosItemResponse response = this.container.replaceItem( - getDocumentDefinition(scenario.id), - scenario.id, - new PartitionKey(scenario.id), - null); - - deserializeAndValidatePayload(response, scenario.id, expected.ExpectedReplaceStatusCode); - } catch (Throwable throwable) { - CosmosException cosmosError = Utils.as(Exceptions.unwrap(throwable), CosmosException.class); - if (cosmosError == null) { - Fail.fail( - "Unexpected exception type " + Exceptions.unwrap(throwable).getClass().getName(), - throwable); + try { + CosmosItemResponse response = this.container.readItem( + scenario.id, + new PartitionKey(scenario.id), + ObjectNode.class); + + deserializeAndValidatePayload(response, scenario.id, expected.ExpectedReadStatusCode); + } catch (Throwable throwable) { + CosmosException cosmosError = Utils.as(Exceptions.unwrap(throwable), CosmosException.class); + if (cosmosError == null) { + if (expected.ExpectedReadStatusCode == -1) { + return; + } + + Fail.fail( + "Unexpected exception type " + Exceptions.unwrap(throwable).getClass().getName(), + throwable); + } + if (cosmosError.getStatusCode() == 0 && + cosmosError.getCause() instanceof IllegalArgumentException && + cosmosError.getCause().getCause() instanceof JsonParseException && + cosmosError.getCause().getCause().toString().contains("Bad Request")) { + + logger.info("HTML BAD REQUEST", cosmosError); + assertThat(expected.ExpectedReadStatusCode).isEqualTo(400); + return; + } else { + logger.info("BAD REQUEST", cosmosError); + assertThat(cosmosError.getStatusCode()).isEqualTo(expected.ExpectedReadStatusCode); + } } - assertThat(cosmosError.getStatusCode()).isEqualTo(expected.ExpectedReplaceStatusCode); - } - try { - CosmosItemResponse response = this.container.deleteItem( - scenario.id, - new PartitionKey(scenario.id), - (CosmosItemRequestOptions)null); - - assertThat(response.getStatusCode()).isEqualTo(expected.ExpectedDeleteStatusCode); - } catch (Throwable throwable) { - CosmosException cosmosError = Utils.as(Exceptions.unwrap(throwable), CosmosException.class); - if (cosmosError == null) { - Fail.fail( - "Unexpected exception type " + Exceptions.unwrap(throwable).getClass().getName(), - throwable); + try { + CosmosItemResponse response = this.container.replaceItem( + getDocumentDefinition(scenario.id), + scenario.id, + new PartitionKey(scenario.id), + null); + + deserializeAndValidatePayload(response, scenario.id, expected.ExpectedReplaceStatusCode); + } catch (Throwable throwable) { + CosmosException cosmosError = Utils.as(Exceptions.unwrap(throwable), CosmosException.class); + if (cosmosError == null) { + Fail.fail( + "Unexpected exception type " + Exceptions.unwrap(throwable).getClass().getName(), + throwable); + } + assertThat(cosmosError.getStatusCode()).isEqualTo(expected.ExpectedReplaceStatusCode); } - assertThat(cosmosError.getStatusCode()).isEqualTo(expected.ExpectedDeleteStatusCode); + + try { + CosmosItemResponse response = this.container.deleteItem( + scenario.id, + new PartitionKey(scenario.id), + (CosmosItemRequestOptions) null); + + assertThat(response.getStatusCode()).isEqualTo(expected.ExpectedDeleteStatusCode); + } catch (Throwable throwable) { + CosmosException cosmosError = Utils.as(Exceptions.unwrap(throwable), CosmosException.class); + if (cosmosError == null) { + Fail.fail( + "Unexpected exception type " + Exceptions.unwrap(throwable).getClass().getName(), + throwable); + } + assertThat(cosmosError.getStatusCode()).isEqualTo(expected.ExpectedDeleteStatusCode); + } + } finally { + System.clearProperty(Configs.PREVENT_INVALID_ID_CHARS); } } @@ -757,17 +796,31 @@ public TestScenario( TestScenarioExpectations computeGateway, TestScenarioExpectations direct) { + this(name, id, false, gateway, computeGateway, direct); + } + + public TestScenario( + String name, + String id, + boolean idValidationEnabled, + TestScenarioExpectations gateway, + TestScenarioExpectations computeGateway, + TestScenarioExpectations direct) { + this.name = name; this.id = id; this.gateway = gateway; this.computeGateway = computeGateway; this.direct = direct; + this.idValidationEnabled = idValidationEnabled; } public String name; public String id; + public boolean idValidationEnabled; + public TestScenarioExpectations gateway; public TestScenarioExpectations computeGateway; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/InternalObjectNodeTest.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/InternalObjectNodeTest.java index 5142523a0bc63..ccb632717482d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/InternalObjectNodeTest.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/InternalObjectNodeTest.java @@ -22,7 +22,8 @@ public class InternalObjectNodeTest { public void PojoWithTrackingId() throws IOException { String expectedTrackingId = UUID.randomUUID().toString(); TestPojo pojo = new TestPojo(); - ByteBuffer buffer = InternalObjectNode.serializeJsonToByteBuffer(pojo, CosmosItemSerializer.DEFAULT_SERIALIZER, expectedTrackingId); + ByteBuffer buffer = InternalObjectNode.serializeJsonToByteBuffer( + pojo, CosmosItemSerializer.DEFAULT_SERIALIZER, expectedTrackingId, false); byte[] blob = new byte[buffer.remaining()]; buffer.get(blob); validateTrackingId(blob, expectedTrackingId); @@ -33,7 +34,8 @@ public void ByteArrayWithTrackingId() throws IOException { String expectedTrackingId = UUID.randomUUID().toString(); ObjectNode objectNode = MAPPER.createObjectNode(); objectNode.put("id", "myId"); - ByteBuffer buffer = InternalObjectNode.serializeJsonToByteBuffer(objectNode, CosmosItemSerializer.DEFAULT_SERIALIZER, expectedTrackingId); + ByteBuffer buffer = InternalObjectNode.serializeJsonToByteBuffer( + objectNode, CosmosItemSerializer.DEFAULT_SERIALIZER, expectedTrackingId, false); byte[] blob = blob = new byte[buffer.remaining()]; buffer.get(blob); validateTrackingId(blob, expectedTrackingId); @@ -44,7 +46,8 @@ public void internalObjectNodeWithTrackingId() throws IOException { String expectedTrackingId = UUID.randomUUID().toString(); InternalObjectNode intenalObjectNode = new InternalObjectNode(); intenalObjectNode.set("id", "myId", CosmosItemSerializer.DEFAULT_SERIALIZER); - ByteBuffer buffer = InternalObjectNode.serializeJsonToByteBuffer(intenalObjectNode, CosmosItemSerializer.DEFAULT_SERIALIZER, expectedTrackingId); + ByteBuffer buffer = InternalObjectNode.serializeJsonToByteBuffer( + intenalObjectNode, CosmosItemSerializer.DEFAULT_SERIALIZER, expectedTrackingId, false); byte[] blob = new byte[buffer.remaining()]; buffer.get(blob); validateTrackingId(blob, expectedTrackingId); @@ -55,7 +58,8 @@ public void objectNodeWithTrackingId() throws IOException { String expectedTrackingId = UUID.randomUUID().toString(); ObjectNode objectNode = MAPPER.createObjectNode(); objectNode.put("id", "myId"); - ByteBuffer buffer = InternalObjectNode.serializeJsonToByteBuffer(objectNode, CosmosItemSerializer.DEFAULT_SERIALIZER, expectedTrackingId); + ByteBuffer buffer = InternalObjectNode.serializeJsonToByteBuffer( + objectNode, CosmosItemSerializer.DEFAULT_SERIALIZER, expectedTrackingId, false); byte[] blob = new byte[buffer.remaining()]; buffer.get(blob); validateTrackingId(blob, expectedTrackingId); diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 1c728918f822b..30ea01807327e 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -3,6 +3,7 @@ ### 4.63.0-beta.1 (Unreleased) #### Features Added +* Added optional id validation to prevent documents with invalid char '/' in id property to be created. - See [PR 41108](https://github.com/Azure/azure-sdk-for-java/pull/41108) * Added support for specifying a set of custom diagnostic correlation ids in the request options. - See [PR 40835](https://github.com/Azure/azure-sdk-for-java/pull/40835) #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java index e5ecc64fb5c04..b706c6781dde5 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Configs.java @@ -189,6 +189,19 @@ public class Configs { public static final String DIAGNOSTICS_PROVIDER_SYSTEM_EXIT_ON_ERROR = "COSMOS.DIAGNOSTICS_PROVIDER_SYSTEM_EXIT_ON_ERROR"; public static final boolean DEFAULT_DIAGNOSTICS_PROVIDER_SYSTEM_EXIT_ON_ERROR = true; + // Out-of-the-box it is possible to create documents with invalid character '/' in the id field + // Client and service will just allow creating these documents - but no read, replace, patch or delete operation + // can be done for these documents because the resulting request uri + // "dbs/DBNAME/cols/CONTAINERNAME/docs/IDVALUE" would become invalid + // Adding a validation to prevent the '/' in the id value would be breaking (for service and client) + // but for some workloads there is a vested interest in failing early if someone tries to create documents + // with invalid id value = the environment variable changes below + // allow opting into a validation client-side. If this becomes used more frequently we might need to create + // a public API for it as well. + public static final String PREVENT_INVALID_ID_CHARS = "COSMOS.PREVENT_INVALID_ID_CHARS"; + public static final String PREVENT_INVALID_ID_CHARS_VARIABLE = "COSMOS_PREVENT_INVALID_ID_CHARS"; + public static final boolean DEFAULT_PREVENT_INVALID_ID_CHARS = false; + // Metrics // Samples: @@ -366,6 +379,20 @@ public static boolean isDefaultE2ETimeoutDisabledForNonPointOperations() { return DEFAULT_E2E_FOR_NON_POINT_DISABLED_DEFAULT; } + public static boolean isIdValueValidationEnabled() { + String valueFromSystemProperty = System.getProperty(PREVENT_INVALID_ID_CHARS); + if (valueFromSystemProperty != null && !valueFromSystemProperty.isEmpty()) { + return !Boolean.valueOf(valueFromSystemProperty); + } + + String valueFromEnvVariable = System.getenv(PREVENT_INVALID_ID_CHARS_VARIABLE); + if (valueFromEnvVariable != null && !valueFromEnvVariable.isEmpty()) { + return!Boolean.valueOf(valueFromEnvVariable); + } + + return DEFAULT_PREVENT_INVALID_ID_CHARS; + } + public static int getMaxHttpRequestTimeout() { String valueFromSystemProperty = System.getProperty(HTTP_MAX_REQUEST_TIMEOUT); if (valueFromSystemProperty != null && !valueFromSystemProperty.isEmpty()) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/HttpConstants.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/HttpConstants.java index 2f91d556ec940..a9be4c70ea8b0 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/HttpConstants.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/HttpConstants.java @@ -387,6 +387,7 @@ public static class SubStatusCodes { // client generated 400s public static final int CUSTOM_SERIALIZER_EXCEPTION = 10101; + public static final int INVALID_ID_VALUE = 10102; // 410: StatusCodeType_Gone: substatus // Merge or split share the same status code and subStatusCode diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/InternalObjectNode.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/InternalObjectNode.java index 58fac984228e9..3c87a8f0993ae 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/InternalObjectNode.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/InternalObjectNode.java @@ -2,6 +2,7 @@ // Licensed under the MIT License. package com.azure.cosmos.implementation; +import com.azure.cosmos.BridgeInternal; import com.azure.cosmos.CosmosItemSerializer; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -107,7 +108,12 @@ public static Document fromObject(Object cosmosItem) { } } - public static ByteBuffer serializeJsonToByteBuffer(Object cosmosItem, CosmosItemSerializer itemSerializer, String trackingId) { + public static ByteBuffer serializeJsonToByteBuffer( + Object cosmosItem, + CosmosItemSerializer itemSerializer, + String trackingId, + boolean isIdValidationEnabled) { + checkNotNull(itemSerializer, "Argument 'itemSerializer' must not be null."); if (cosmosItem instanceof InternalObjectNode) { InternalObjectNode internalObjectNode = ((InternalObjectNode) cosmosItem); @@ -115,25 +121,28 @@ public static ByteBuffer serializeJsonToByteBuffer(Object cosmosItem, CosmosItem if (trackingId != null) { onAfterSerialization = (node) -> node.put(Constants.Properties.TRACKING_ID, trackingId); } - return internalObjectNode.serializeJsonToByteBuffer(itemSerializer, onAfterSerialization); + return internalObjectNode.serializeJsonToByteBuffer(itemSerializer, onAfterSerialization, isIdValidationEnabled); } else if (cosmosItem instanceof Document) { Document doc = (Document) cosmosItem; Consumer> onAfterSerialization = null; if (trackingId != null) { onAfterSerialization = (node) -> node.put(Constants.Properties.TRACKING_ID, trackingId); } - return doc.serializeJsonToByteBuffer(itemSerializer, onAfterSerialization); + return doc.serializeJsonToByteBuffer(itemSerializer, onAfterSerialization, isIdValidationEnabled); } else if (cosmosItem instanceof ObjectNode) { ObjectNode objectNode = (ObjectNode)cosmosItem; Consumer> onAfterSerialization = null; if (trackingId != null) { onAfterSerialization = (node) -> node.put(Constants.Properties.TRACKING_ID, trackingId); } - return (new InternalObjectNode(objectNode).serializeJsonToByteBuffer(itemSerializer, onAfterSerialization)); + return (new InternalObjectNode(objectNode).serializeJsonToByteBuffer(itemSerializer, onAfterSerialization, isIdValidationEnabled)); } else if (cosmosItem instanceof byte[]) { if (trackingId != null) { InternalObjectNode internalObjectNode = new InternalObjectNode((byte[]) cosmosItem); - return internalObjectNode.serializeJsonToByteBuffer(itemSerializer, (node) -> node.put(Constants.Properties.TRACKING_ID, trackingId)); + return internalObjectNode.serializeJsonToByteBuffer( + itemSerializer, + (node) -> node.put(Constants.Properties.TRACKING_ID, trackingId), + isIdValidationEnabled); } return ByteBuffer.wrap((byte[]) cosmosItem); } else { @@ -142,7 +151,7 @@ public static ByteBuffer serializeJsonToByteBuffer(Object cosmosItem, CosmosItem onAfterSerialization = (node) -> node.put(Constants.Properties.TRACKING_ID, trackingId); } - return Utils.serializeJsonToByteBuffer(itemSerializer, cosmosItem, onAfterSerialization); + return Utils.serializeJsonToByteBuffer(itemSerializer, cosmosItem, onAfterSerialization, isIdValidationEnabled); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/JsonSerializable.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/JsonSerializable.java index c834518bb21ac..d0906c9a67662 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/JsonSerializable.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/JsonSerializable.java @@ -656,9 +656,9 @@ private ObjectNode fromJson(ByteBuffer json) { } } - public ByteBuffer serializeJsonToByteBuffer(CosmosItemSerializer itemSerializer, Consumer> onAfterSerialization) { + public ByteBuffer serializeJsonToByteBuffer(CosmosItemSerializer itemSerializer, Consumer> onAfterSerialization, boolean isIdValidationEnabled) { this.populatePropertyBag(); - return Utils.serializeJsonToByteBuffer(itemSerializer, propertyBag, onAfterSerialization); + return Utils.serializeJsonToByteBuffer(itemSerializer, propertyBag, onAfterSerialization, isIdValidationEnabled); } private String toJson(Object object) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java index b07ae2a28538f..9daa031af71b2 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java @@ -907,7 +907,7 @@ private Mono> createDatabaseInternal(Database databas Map requestHeaders = this.getRequestHeaders(options, ResourceType.Database, OperationType.Create); Instant serializationStartTimeUTC = Instant.now(); - ByteBuffer byteBuffer = database.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + ByteBuffer byteBuffer = database.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null, false); Instant serializationEndTimeUTC = Instant.now(); SerializationDiagnosticsContext.SerializationDiagnostics serializationDiagnostics = new SerializationDiagnosticsContext.SerializationDiagnostics( serializationStartTimeUTC, @@ -1314,7 +1314,7 @@ private Mono> createCollectionInternal(Stri Map requestHeaders = this.getRequestHeaders(options, ResourceType.DocumentCollection, OperationType.Create); Instant serializationStartTimeUTC = Instant.now(); - ByteBuffer byteBuffer = collection.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + ByteBuffer byteBuffer = collection.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null, false); Instant serializationEndTimeUTC = Instant.now(); SerializationDiagnosticsContext.SerializationDiagnostics serializationDiagnostics = new SerializationDiagnosticsContext.SerializationDiagnostics( serializationStartTimeUTC, @@ -1367,7 +1367,7 @@ private Mono> replaceCollectionInternal(Doc String path = Utils.joinPath(collection.getSelfLink(), null); Map requestHeaders = this.getRequestHeaders(options, ResourceType.DocumentCollection, OperationType.Replace); Instant serializationStartTimeUTC = Instant.now(); - ByteBuffer byteBuffer = collection.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + ByteBuffer byteBuffer = collection.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null, false); Instant serializationEndTimeUTC = Instant.now(); SerializationDiagnosticsContext.SerializationDiagnostics serializationDiagnostics = new SerializationDiagnosticsContext.SerializationDiagnostics( serializationStartTimeUTC, @@ -1814,7 +1814,7 @@ private Mono getCreateDocumentRequest(DocumentClientRe if (options != null) { trackingId = options.getTrackingId(); } - ByteBuffer content = InternalObjectNode.serializeJsonToByteBuffer(document, options.getEffectiveItemSerializer(), trackingId); + ByteBuffer content = InternalObjectNode.serializeJsonToByteBuffer(document, options.getEffectiveItemSerializer(), trackingId, true); Instant serializationEndTimeUTC = Instant.now(); SerializationDiagnosticsContext.SerializationDiagnostics serializationDiagnostics = new SerializationDiagnosticsContext.SerializationDiagnostics( @@ -2586,7 +2586,7 @@ private Mono> replaceDocumentInternal( } } - ByteBuffer content = document.serializeJsonToByteBuffer(options.getEffectiveItemSerializer(), onAfterSerialization); + ByteBuffer content = document.serializeJsonToByteBuffer(options.getEffectiveItemSerializer(), onAfterSerialization, false); Instant serializationEndTime = Instant.now(); SerializationDiagnosticsContext.SerializationDiagnostics serializationDiagnostics = new SerializationDiagnosticsContext.SerializationDiagnostics( diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentServiceRequest.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentServiceRequest.java index 4dc7c3cded72d..5d39d58914ad9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentServiceRequest.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentServiceRequest.java @@ -389,7 +389,12 @@ public static RxDocumentServiceRequest create(DiagnosticsClientContext clientCon Object options) { RxDocumentServiceRequest request = new RxDocumentServiceRequest(clientContext, operation, resourceType, relativePath, - resource.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null), headers, AuthorizationTokenType.PrimaryMasterKey); + resource.serializeJsonToByteBuffer( + CosmosItemSerializer.DEFAULT_SERIALIZER, + null, + resourceType == ResourceType.Document && (operation == OperationType.Create || operation == OperationType.Upsert)), + headers, + AuthorizationTokenType.PrimaryMasterKey); request.properties = getProperties(options); request.throughputControlGroupName = getThroughputControlGroupName(options); return request; @@ -574,7 +579,10 @@ public static RxDocumentServiceRequest create(DiagnosticsClientContext clientCon ResourceType resourceType, String relativePath, Map headers) { - ByteBuffer resourceContent = resource.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + ByteBuffer resourceContent = resource.serializeJsonToByteBuffer( + CosmosItemSerializer.DEFAULT_SERIALIZER, + null, + resourceType == ResourceType.Document && (operation == OperationType.Create || operation == OperationType.Upsert)); return new RxDocumentServiceRequest(clientContext, operation, resourceType, relativePath, resourceContent, headers, AuthorizationTokenType.PrimaryMasterKey); } @@ -595,7 +603,10 @@ public static RxDocumentServiceRequest create(DiagnosticsClientContext clientCon String relativePath, Map headers, AuthorizationTokenType authorizationTokenType) { - ByteBuffer resourceContent = resource.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + ByteBuffer resourceContent = resource.serializeJsonToByteBuffer( + CosmosItemSerializer.DEFAULT_SERIALIZER, + null, + resourceType == ResourceType.Document && (operation == OperationType.Create || operation == OperationType.Upsert)); return new RxDocumentServiceRequest(clientContext, operation, resourceType, relativePath, resourceContent, headers, authorizationTokenType); } @@ -650,7 +661,10 @@ public static RxDocumentServiceRequest create(DiagnosticsClientContext clientCon ResourceType resourceType, Resource resource, Map headers) { - ByteBuffer resourceContent = resource.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + ByteBuffer resourceContent = resource.serializeJsonToByteBuffer( + CosmosItemSerializer.DEFAULT_SERIALIZER, + null, + resourceType == ResourceType.Document && (operation == OperationType.Create || operation == OperationType.Upsert)); return new RxDocumentServiceRequest(clientContext, operation, resourceId, resourceType, resourceContent, headers, false, AuthorizationTokenType.PrimaryMasterKey); } @@ -671,7 +685,10 @@ public static RxDocumentServiceRequest create(DiagnosticsClientContext clientCon Resource resource, Map headers, AuthorizationTokenType authorizationTokenType) { - ByteBuffer resourceContent = resource.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + ByteBuffer resourceContent = resource.serializeJsonToByteBuffer( + CosmosItemSerializer.DEFAULT_SERIALIZER, + null, + resourceType == ResourceType.Document && (operation == OperationType.Create || operation == OperationType.Upsert)); return new RxDocumentServiceRequest(clientContext, operation, resourceId, resourceType, resourceContent, headers, false, authorizationTokenType); } @@ -725,7 +742,10 @@ public static RxDocumentServiceRequest createFromName( Resource resource, String resourceFullName, ResourceType resourceType) { - ByteBuffer resourceContent = resource.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + ByteBuffer resourceContent = resource.serializeJsonToByteBuffer( + CosmosItemSerializer.DEFAULT_SERIALIZER, + null, + resourceType == ResourceType.Document && (operationType == OperationType.Create || operationType == OperationType.Upsert)); return new RxDocumentServiceRequest(clientContext, operationType, resourceFullName, @@ -744,7 +764,10 @@ public static RxDocumentServiceRequest createFromName( String resourceFullName, ResourceType resourceType, AuthorizationTokenType authorizationTokenType) { - ByteBuffer resourceContent = resource.serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + ByteBuffer resourceContent = resource.serializeJsonToByteBuffer( + CosmosItemSerializer.DEFAULT_SERIALIZER, + null, + resourceType == ResourceType.Document && (operationType == OperationType.Create || operationType == OperationType.Upsert)); return new RxDocumentServiceRequest(clientContext, operationType, resourceFullName, diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java index 53568471398b2..4a32e9eaa8b14 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java @@ -2,6 +2,7 @@ // Licensed under the MIT License. package com.azure.cosmos.implementation; +import com.azure.cosmos.BridgeInternal; import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.CosmosItemSerializer; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; @@ -633,8 +634,34 @@ public static T parse(ObjectNode jsonNode, Class itemClassType, CosmosIte return ensureItemSerializerAccessor().deserializeSafe(effectiveItemSerializer, new ObjectNodeMap(jsonNode), itemClassType); } + public static void validateIdValue(Object itemIdValue) { + if (!(itemIdValue instanceof String)) { + return; + } + + String itemId = (String)itemIdValue; + if (itemId != null + && Configs.isIdValueValidationEnabled() + && itemId.contains("/")) { + + BadRequestException exception = new BadRequestException( + "The id value '" + itemId + "' contains the invalid character '/'. To stop the client-side validation " + + "set the environment variable '" + Configs.PREVENT_INVALID_ID_CHARS + "' or the system property '" + + Configs.PREVENT_INVALID_ID_CHARS_VARIABLE + "' to 'true'."); + + BridgeInternal.setSubStatusCode(exception, HttpConstants.SubStatusCodes.INVALID_ID_VALUE); + + throw exception; + } + } + @SuppressWarnings("unchecked") - public static ByteBuffer serializeJsonToByteBuffer(CosmosItemSerializer serializer, Object object, Consumer> onAfterSerialization) { + public static ByteBuffer serializeJsonToByteBuffer( + CosmosItemSerializer serializer, + Object object, + Consumer> onAfterSerialization, + boolean isIdValidationEnabled) { + checkArgument(serializer != null || object instanceof Map, "Argument 'serializer' must not be null."); try { ByteBufferOutputStream byteBufferOutputStream = new ByteBufferOutputStream(ONE_KB); @@ -642,6 +669,10 @@ public static ByteBuffer serializeJsonToByteBuffer(CosmosItemSerializer serializ ? (Map) object : ensureItemSerializerAccessor().serializeSafe(serializer, object); + if (isIdValidationEnabled) { + validateIdValue(jsonTreeMap.get(Constants.Properties.ID)); + } + if (onAfterSerialization != null) { onAfterSerialization.accept(jsonTreeMap); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/clienttelemetry/ClientTelemetry.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/clienttelemetry/ClientTelemetry.java index 855081b4c204f..4027dee776ace 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/clienttelemetry/ClientTelemetry.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/clienttelemetry/ClientTelemetry.java @@ -273,7 +273,8 @@ private Mono sendClientTelemetry() { ByteBuffer byteBuffer = InternalObjectNode.serializeJsonToByteBuffer(this.clientTelemetryInfo, CosmosItemSerializer.DEFAULT_SERIALIZER, - null); + null, + false); byte[] tempBuffer = RxDocumentServiceRequest.toByteArray(byteBuffer); Map headers = new HashMap<>(); String date = Utils.nowAsRFC1123(); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java index 8bb3ca4fa7804..5876e15196695 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java @@ -402,7 +402,7 @@ public static CosmosChangeFeedRequestOptions getEffectiveChangeFeedRequestOption @Warning(value = INTERNAL_USE_ONLY_WARNING) public static ByteBuffer serializeJsonToByteBuffer(SqlQuerySpec sqlQuerySpec) { sqlQuerySpec.populatePropertyBag(); - return sqlQuerySpec.getJsonSerializable().serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null); + return sqlQuerySpec.getJsonSerializable().serializeJsonToByteBuffer(CosmosItemSerializer.DEFAULT_SERIALIZER, null, false); } @Warning(value = INTERNAL_USE_ONLY_WARNING)