diff --git a/Microsoft.Azure.Cosmos/src/GatewayStoreModel.cs b/Microsoft.Azure.Cosmos/src/GatewayStoreModel.cs index e299af088a..d0690d4d4a 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayStoreModel.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayStoreModel.cs @@ -369,7 +369,6 @@ private static async Task> TryResolvePartitionKey collectionRid: collection.ResourceId, previousValue: null, request: request, - cancellationToken: CancellationToken.None, NoOpTrace.Singleton); if (refreshCache && collectionRoutingMap != null) @@ -378,7 +377,6 @@ private static async Task> TryResolvePartitionKey collectionRid: collection.ResourceId, previousValue: collectionRoutingMap, request: request, - cancellationToken: CancellationToken.None, NoOpTrace.Singleton); } diff --git a/Microsoft.Azure.Cosmos/src/Query/v2Query/PartitionKeyRangeGoneRetryPolicy.cs b/Microsoft.Azure.Cosmos/src/Query/v2Query/PartitionKeyRangeGoneRetryPolicy.cs index aa9c1607e4..de62804456 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v2Query/PartitionKeyRangeGoneRetryPolicy.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v2Query/PartitionKeyRangeGoneRetryPolicy.cs @@ -121,16 +121,20 @@ private async Task ShouldRetryInternalAsync( AuthorizationTokenType.PrimaryMasterKey)) { ContainerProperties collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken, this.trace); - CollectionRoutingMap routingMap = await this.partitionKeyRangeCache.TryLookupAsync(collection.ResourceId, null, request, cancellationToken, this.trace); + CollectionRoutingMap routingMap = await this.partitionKeyRangeCache.TryLookupAsync( + collectionRid: collection.ResourceId, + previousValue: null, + request: request, + trace: this.trace); + if (routingMap != null) { // Force refresh. await this.partitionKeyRangeCache.TryLookupAsync( - collection.ResourceId, - routingMap, - request, - cancellationToken, - this.trace); + collectionRid: collection.ResourceId, + previousValue: routingMap, + request: request, + trace: this.trace); } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs index 6dc1249a1b..48b22ef402 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.cs @@ -508,7 +508,6 @@ public override async Task GetRoutingMapAsync(Cancellation collectionRid, previousValue: null, request: null, - cancellationToken, NoOpTrace.Singleton); // Not found. @@ -523,7 +522,6 @@ public override async Task GetRoutingMapAsync(Cancellation collectionRid, previousValue: null, request: null, - cancellationToken, NoOpTrace.Singleton); } diff --git a/Microsoft.Azure.Cosmos/src/Routing/AddressResolver.cs b/Microsoft.Azure.Cosmos/src/Routing/AddressResolver.cs index 271068302f..66a4e6f6b7 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/AddressResolver.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/AddressResolver.cs @@ -263,7 +263,10 @@ private async Task ResolveAddressesAndIdentityAsync( ContainerProperties collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken, NoOpTrace.Singleton); CollectionRoutingMap routingMap = await this.collectionRoutingMapCache.TryLookupAsync( - collection.ResourceId, null, request, cancellationToken, NoOpTrace.Singleton); + collectionRid: collection.ResourceId, + previousValue: null, + request: request, + trace: NoOpTrace.Singleton); if (routingMap != null && request.ForceCollectionRoutingMapRefresh) { @@ -271,7 +274,11 @@ private async Task ResolveAddressesAndIdentityAsync( "AddressResolver.ResolveAddressesAndIdentityAsync ForceCollectionRoutingMapRefresh collection.ResourceId = {0}", collection.ResourceId); - routingMap = await this.collectionRoutingMapCache.TryLookupAsync(collection.ResourceId, routingMap, request, cancellationToken, NoOpTrace.Singleton); + routingMap = await this.collectionRoutingMapCache.TryLookupAsync( + collectionRid: collection.ResourceId, + previousValue: routingMap, + request: request, + trace: NoOpTrace.Singleton); } if (request.ForcePartitionKeyRangeRefresh) @@ -280,7 +287,11 @@ private async Task ResolveAddressesAndIdentityAsync( request.ForcePartitionKeyRangeRefresh = false; if (routingMap != null) { - routingMap = await this.collectionRoutingMapCache.TryLookupAsync(collection.ResourceId, routingMap, request, cancellationToken, NoOpTrace.Singleton); + routingMap = await this.collectionRoutingMapCache.TryLookupAsync( + collectionRid: collection.ResourceId, + previousValue: routingMap, + request: request, + trace: NoOpTrace.Singleton); } } @@ -293,10 +304,9 @@ private async Task ResolveAddressesAndIdentityAsync( collectionRoutingMapCacheIsUptoDate = false; collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken, NoOpTrace.Singleton); routingMap = await this.collectionRoutingMapCache.TryLookupAsync( - collection.ResourceId, + collectionRid: collection.ResourceId, previousValue: null, request: request, - cancellationToken: cancellationToken, trace: NoOpTrace.Singleton); } @@ -319,7 +329,6 @@ private async Task ResolveAddressesAndIdentityAsync( if (!collectionCacheIsUptoDate) { request.ForceNameCacheRefresh = true; - collectionCacheIsUptoDate = true; collection = await this.collectionCache.ResolveCollectionAsync(request, cancellationToken, NoOpTrace.Singleton); if (collection.ResourceId != routingMap.CollectionUniqueId) { @@ -327,22 +336,19 @@ private async Task ResolveAddressesAndIdentityAsync( // for this new collection rid. Mark it as such. collectionRoutingMapCacheIsUptoDate = false; routingMap = await this.collectionRoutingMapCache.TryLookupAsync( - collection.ResourceId, + collectionRid: collection.ResourceId, previousValue: null, request: request, - cancellationToken: cancellationToken, trace: NoOpTrace.Singleton); } } if (!collectionRoutingMapCacheIsUptoDate) { - collectionRoutingMapCacheIsUptoDate = true; routingMap = await this.collectionRoutingMapCache.TryLookupAsync( collection.ResourceId, previousValue: routingMap, request: request, - cancellationToken: cancellationToken, trace: NoOpTrace.Singleton); } @@ -449,7 +455,6 @@ private async Task TryResolveServerPartitionAsync( PartitionKeyRange range; string partitionKeyString = request.Headers[HttpConstants.HttpHeaders.PartitionKey]; - object effectivePartitionKeyStringObject = null; if (partitionKeyString != null) { range = AddressResolver.TryResolveServerPartitionByPartitionKey( @@ -461,7 +466,7 @@ private async Task TryResolveServerPartitionAsync( } else if (request.Properties != null && request.Properties.TryGetValue( WFConstants.BackendHeaders.EffectivePartitionKeyString, - out effectivePartitionKeyStringObject)) + out object effectivePartitionKeyStringObject)) { // Allow EPK only for partitioned collection (excluding migrated fixed collections) if (!collection.HasPartitionKey || collection.PartitionKey.IsSystemKey.GetValueOrDefault(false)) diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs index 23d17c84bc..d27d84bdcf 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GlobalAddressResolver.cs @@ -82,8 +82,11 @@ public async Task OpenAsync( ContainerProperties collection, CancellationToken cancellationToken) { - CollectionRoutingMap routingMap = - await this.routingMapProvider.TryLookupAsync(collection.ResourceId, null, null, cancellationToken, NoOpTrace.Singleton); + CollectionRoutingMap routingMap = await this.routingMapProvider.TryLookupAsync( + collectionRid: collection.ResourceId, + previousValue: null, + request: null, + trace: NoOpTrace.Singleton); if (routingMap == null) { diff --git a/Microsoft.Azure.Cosmos/src/Routing/ICollectionRoutingMapCache.cs b/Microsoft.Azure.Cosmos/src/Routing/ICollectionRoutingMapCache.cs index b2b3727e44..42fc8f829f 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/ICollectionRoutingMapCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/ICollectionRoutingMapCache.cs @@ -15,7 +15,6 @@ Task TryLookupAsync( string collectionRid, CollectionRoutingMap previousValue, DocumentServiceRequest request, - CancellationToken cancellationToken, ITrace trace); } } diff --git a/Microsoft.Azure.Cosmos/src/Routing/PartitionKeyRangeCache.cs b/Microsoft.Azure.Cosmos/src/Routing/PartitionKeyRangeCache.cs index bcc7a3df12..7c28dd2d2a 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/PartitionKeyRangeCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/PartitionKeyRangeCache.cs @@ -25,7 +25,7 @@ internal class PartitionKeyRangeCache : IRoutingMapProvider, ICollectionRoutingM { private const string PageSizeString = "-1"; - private readonly AsyncCache routingMapCache; + private readonly AsyncCacheNonBlocking routingMapCache; private readonly ICosmosAuthorizationTokenProvider authorizationTokenProvider; private readonly IStoreModel storeModel; @@ -36,9 +36,8 @@ public PartitionKeyRangeCache( IStoreModel storeModel, CollectionCache collectionCache) { - this.routingMapCache = new AsyncCache( - EqualityComparer.Default, - StringComparer.Ordinal); + this.routingMapCache = new AsyncCacheNonBlocking( + keyEqualityComparer: StringComparer.Ordinal); this.authorizationTokenProvider = authorizationTokenProvider; this.storeModel = storeModel; this.collectionCache = collectionCache; @@ -54,12 +53,19 @@ public virtual async Task> TryGetOverlappingRan { Debug.Assert(ResourceId.TryParse(collectionRid, out ResourceId collectionRidParsed), "Could not parse CollectionRid from ResourceId."); - CollectionRoutingMap routingMap = - await this.TryLookupAsync(collectionRid, null, null, CancellationToken.None, childTrace); + CollectionRoutingMap routingMap = await this.TryLookupAsync( + collectionRid: collectionRid, + previousValue: null, + request: null, + trace: childTrace); if (forceRefresh && routingMap != null) { - routingMap = await this.TryLookupAsync(collectionRid, routingMap, null, CancellationToken.None, childTrace); + routingMap = await this.TryLookupAsync( + collectionRid: collectionRid, + previousValue: routingMap, + request: null, + trace: childTrace); } if (routingMap == null) @@ -78,15 +84,21 @@ public virtual async Task TryGetPartitionKeyRangeByIdAsync( ITrace trace, bool forceRefresh = false) { - ResourceId collectionRidParsed; - Debug.Assert(ResourceId.TryParse(collectionResourceId, out collectionRidParsed), "Could not parse CollectionRid from ResourceId."); + Debug.Assert(ResourceId.TryParse(collectionResourceId, out _), "Could not parse CollectionRid from ResourceId."); - CollectionRoutingMap routingMap = - await this.TryLookupAsync(collectionResourceId, null, null, CancellationToken.None, trace); + CollectionRoutingMap routingMap = await this.TryLookupAsync( + collectionRid: collectionResourceId, + previousValue: null, + request: null, + trace: trace); if (forceRefresh && routingMap != null) { - routingMap = await this.TryLookupAsync(collectionResourceId, routingMap, null, CancellationToken.None, trace); + routingMap = await this.TryLookupAsync( + collectionRid: collectionResourceId, + previousValue: routingMap, + request: null, + trace: trace); } if (routingMap == null) @@ -102,20 +114,19 @@ public virtual async Task TryLookupAsync( string collectionRid, CollectionRoutingMap previousValue, DocumentServiceRequest request, - CancellationToken cancellationToken, ITrace trace) { try { return await this.routingMapCache.GetAsync( - collectionRid, - previousValue, - () => this.GetRoutingMapForCollectionAsync(collectionRid, - previousValue, - trace, - request?.RequestContext?.ClientRequestStatistics, - cancellationToken), - CancellationToken.None); + key: collectionRid, + singleValueInitFunc: () => this.GetRoutingMapForCollectionAsync( + collectionRid, + previousValue, + trace, + request?.RequestContext?.ClientRequestStatistics), + forceRefresh: (currentValue) => PartitionKeyRangeCache.ShouldForceRefresh(previousValue, currentValue), + callBackOnForceRefresh: null); } catch (DocumentClientException ex) { @@ -139,6 +150,31 @@ public virtual async Task TryLookupAsync( } } + private static bool ShouldForceRefresh( + CollectionRoutingMap previousValue, + CollectionRoutingMap currentValue) + { + // Previous is null then no need to force a refresh + // The request didn't access the cache before + if (previousValue == null) + { + return false; + } + + // currentValue is null then the value just got initialized so + // is not possible for it to be stale + if (currentValue == null) + { + return false; + } + + // CollectionRoutingMap uses changefeed to update the cache. The ChangeFeedNextIfNoneMatch + // is the continuation token for the changefeed operation. If the values do not match + // then another operation has already refresh the cache since this request was sent. So + // there is no reason to do another refresh. + return previousValue.ChangeFeedNextIfNoneMatch == currentValue.ChangeFeedNextIfNoneMatch; + } + public async Task TryGetRangeByPartitionKeyRangeIdAsync(string collectionRid, string partitionKeyRangeId, ITrace trace, @@ -147,10 +183,14 @@ public async Task TryGetRangeByPartitionKeyRangeIdAsync(strin try { CollectionRoutingMap routingMap = await this.routingMapCache.GetAsync( - collectionRid, - null, - () => this.GetRoutingMapForCollectionAsync(collectionRid, null, trace, clientSideRequestStatistics, CancellationToken.None), - CancellationToken.None); + key: collectionRid, + singleValueInitFunc: () => this.GetRoutingMapForCollectionAsync( + collectionRid: collectionRid, + previousRoutingMap: null, + trace: trace, + clientSideRequestStatistics: clientSideRequestStatistics), + forceRefresh: (_) => false, + callBackOnForceRefresh: null); return routingMap.TryGetRangeByPartitionKeyRangeId(partitionKeyRangeId); } @@ -169,11 +209,10 @@ private async Task GetRoutingMapForCollectionAsync( string collectionRid, CollectionRoutingMap previousRoutingMap, ITrace trace, - IClientSideRequestStatistics clientSideRequestStatistics, - CancellationToken cancellationToken) + IClientSideRequestStatistics clientSideRequestStatistics) { List ranges = new List(); - string changeFeedNextIfNoneMatch = previousRoutingMap == null ? null : previousRoutingMap.ChangeFeedNextIfNoneMatch; + string changeFeedNextIfNoneMatch = previousRoutingMap?.ChangeFeedNextIfNoneMatch; HttpStatusCode lastStatusCode = HttpStatusCode.OK; do @@ -190,8 +229,7 @@ private async Task GetRoutingMapForCollectionAsync( RetryOptions retryOptions = new RetryOptions(); using (DocumentServiceResponse response = await BackoffRetryUtility.ExecuteAsync( () => this.ExecutePartitionKeyRangeReadChangeFeedAsync(collectionRid, headers, trace, clientSideRequestStatistics), - new ResourceThrottleRetryPolicy(retryOptions.MaxRetryAttemptsOnThrottledRequests, retryOptions.MaxRetryWaitTimeInSeconds), - cancellationToken)) + new ResourceThrottleRetryPolicy(retryOptions.MaxRetryAttemptsOnThrottledRequests, retryOptions.MaxRetryWaitTimeInSeconds))) { lastStatusCode = response.StatusCode; changeFeedNextIfNoneMatch = response.Headers[HttpConstants.HttpHeaders.ETag]; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/PartitionKeyRangeCacheTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/PartitionKeyRangeCacheTests.cs index a9a7bb16da..d34a516810 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/PartitionKeyRangeCacheTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/PartitionKeyRangeCacheTests.cs @@ -3,17 +3,235 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests { using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; using System.Net; + using System.Net.Http; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Cosmos.Tracing; using Microsoft.VisualStudio.TestTools.UnitTesting; + using static Microsoft.Azure.Cosmos.SDK.EmulatorTests.TransportClientHelper; [TestClass] public class PartitionKeyRangeCacheTests { + private bool loopBackgroundOperaitons = false; + + [TestMethod] + public async Task VerifyPkRangeCacheRefreshOnSplitWithErrorsAsync() + { + this.loopBackgroundOperaitons = false; + + int throwOnPkRefreshCount = 3; + int pkRangeCalls = 0; + bool causeSplitExceptionInRntbdCall = false; + HttpClientHandlerHelper httpHandlerHelper = new(); + List ifNoneMatchValues = new(); + string failedIfNoneMatchValue = null; + httpHandlerHelper.RequestCallBack = (request, cancellationToken) => + { + if (!request.RequestUri.ToString().EndsWith("pkranges")) + { + return null; + } + + ifNoneMatchValues.Add(request.Headers.IfNoneMatch.ToString()); + + pkRangeCalls++; + + if (pkRangeCalls == throwOnPkRefreshCount) + { + failedIfNoneMatchValue = request.Headers.IfNoneMatch.ToString(); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.InternalServerError)); + } + + return null; + }; + + int countSplitExceptions = 0; + CosmosClientOptions clientOptions = new CosmosClientOptions() + { + HttpClientFactory = () => new HttpClient(httpHandlerHelper), + TransportClientHandlerFactory = (transportClient) => new TransportClientWrapper( + transportClient, + (uri, resource, dsr) => + { + if (dsr.OperationType == Documents.OperationType.Read && + dsr.ResourceType == Documents.ResourceType.Document && + causeSplitExceptionInRntbdCall) + { + countSplitExceptions++; + causeSplitExceptionInRntbdCall = false; + throw new Documents.Routing.PartitionKeyRangeIsSplittingException("Test"); + } + }) + }; + + CosmosClient resourceClient = TestCommon.CreateCosmosClient(clientOptions); + + string dbName = Guid.NewGuid().ToString(); + string containerName = nameof(PartitionKeyRangeCacheTests); + + Database db = await resourceClient.CreateDatabaseIfNotExistsAsync(dbName); + Container container = await db.CreateContainerIfNotExistsAsync( + containerName, + "/pk", + 400); + + // Start a background job that loops forever + List exceptions = new(); + Task backgroundItemOperatios = Task.Factory.StartNew(() => this.CreateAndReadItemBackgroundLoop(container, exceptions)); + + // Wait for the background job to start + Stopwatch stopwatch = Stopwatch.StartNew(); + while (!this.loopBackgroundOperaitons && stopwatch.Elapsed.TotalSeconds < 30) + { + await Task.Delay(TimeSpan.FromSeconds(.5)); + } + + Assert.IsTrue(this.loopBackgroundOperaitons); + Assert.AreEqual(2, pkRangeCalls); + + // Cause direct call to hit a split exception and wait for the background job to hit it + causeSplitExceptionInRntbdCall = true; + stopwatch = Stopwatch.StartNew(); + while (causeSplitExceptionInRntbdCall && stopwatch.Elapsed.TotalSeconds < 10) + { + await Task.Delay(TimeSpan.FromSeconds(.5)); + } + Assert.IsFalse(causeSplitExceptionInRntbdCall); + Assert.AreEqual(3, pkRangeCalls); + + // Cause another direct call split exception + causeSplitExceptionInRntbdCall = true; + stopwatch = Stopwatch.StartNew(); + while (causeSplitExceptionInRntbdCall && stopwatch.Elapsed.TotalSeconds < 10) + { + await Task.Delay(TimeSpan.FromSeconds(.5)); + } + + Assert.IsFalse(causeSplitExceptionInRntbdCall); + + Assert.AreEqual(4, pkRangeCalls); + + Assert.AreEqual(1, ifNoneMatchValues.Count(x => string.IsNullOrEmpty(x))); + Assert.AreEqual(3, ifNoneMatchValues.Count(x => x == failedIfNoneMatchValue), $"3 request with same if none value. 1 initial, 2 from the split errors. split exception count: {countSplitExceptions}; {string.Join(';', ifNoneMatchValues)}"); + + HashSet verifyUniqueIfNoneHeaderValues = new HashSet(); + foreach (string ifNoneValue in ifNoneMatchValues) + { + if (!verifyUniqueIfNoneHeaderValues.Contains(ifNoneValue)) + { + verifyUniqueIfNoneHeaderValues.Add(ifNoneValue); + } + else if (ifNoneValue != failedIfNoneMatchValue) + { + Assert.AreEqual(failedIfNoneMatchValue, ifNoneValue, $"Multiple duplicates; split exception count: {countSplitExceptions}; {string.Join(';', ifNoneMatchValues)}"); + } + } + + Assert.AreEqual(0, exceptions.Count, $"Unexpected exceptions: {string.Join(';', exceptions)}"); + } + + private async Task CreateAndReadItemBackgroundLoop(Container container, List exceptions) + { + this.loopBackgroundOperaitons = true; + + while (this.loopBackgroundOperaitons) + { + ToDoActivity toDoActivity = ToDoActivity.CreateRandomToDoActivity(); + try + { + await container.CreateItemAsync(toDoActivity); + await container.ReadItemAsync(toDoActivity.id, new PartitionKey(toDoActivity.pk)); + } + catch (CosmosException ce) when (ce.StatusCode == HttpStatusCode.InternalServerError) + { + // Expected exception caused by failure on pk range refresh + } + catch (Exception ex) + { + exceptions.Add(ex); + } + } + } + + [TestMethod] + public async Task VerifyPkRangeCacheRefreshOnThrottlesAsync() + { + int pkRangeCalls = 0; + bool causeSplitExceptionInRntbdCall = false; + HttpClientHandlerHelper httpHandlerHelper = new(); + List ifNoneMatchValues = new(); + httpHandlerHelper.RequestCallBack = (request, cancellationToken) => + { + if (!request.RequestUri.ToString().EndsWith("pkranges")) + { + return null; + } + + ifNoneMatchValues.Add(request.Headers.IfNoneMatch.ToString()); + + pkRangeCalls++; + + // Cause throttle on the init call + if (pkRangeCalls <= 3) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.TooManyRequests)); + } + + return null; + }; + + int countSplitExceptions = 0; + CosmosClientOptions clientOptions = new CosmosClientOptions() + { + HttpClientFactory = () => new HttpClient(httpHandlerHelper), + TransportClientHandlerFactory = (transportClient) => new TransportClientWrapper( + transportClient, + (uri, resource, dsr) => + { + if (dsr.ResourceType == Documents.ResourceType.Document && + causeSplitExceptionInRntbdCall) + { + countSplitExceptions++; + causeSplitExceptionInRntbdCall = false; + throw new Documents.Routing.PartitionKeyRangeIsSplittingException("Test"); + } + }) + }; + + CosmosClient resourceClient = TestCommon.CreateCosmosClient(clientOptions); + + string dbName = Guid.NewGuid().ToString(); + string containerName = nameof(PartitionKeyRangeCacheTests); + + Database db = await resourceClient.CreateDatabaseIfNotExistsAsync(dbName); + Container container = await db.CreateContainerIfNotExistsAsync( + containerName, + "/pk", + 400); + + ToDoActivity toDoActivity = ToDoActivity.CreateRandomToDoActivity(); + await container.CreateItemAsync(toDoActivity); + Assert.AreEqual(5, pkRangeCalls); + + Assert.AreEqual(4, ifNoneMatchValues.Count(x => string.IsNullOrEmpty(x)), "First 3 calls are throttled and 4 succeeds"); + + string lastIfNoneMatchValue = ifNoneMatchValues.Last(); + ifNoneMatchValues.Clear(); + + pkRangeCalls = 0; + causeSplitExceptionInRntbdCall = true; + await container.ReadItemAsync(toDoActivity.id, new PartitionKey(toDoActivity.pk)); + Assert.AreEqual(4, pkRangeCalls); + + Assert.AreEqual(0, ifNoneMatchValues.Count(x => string.IsNullOrEmpty(x)), "The cache is already init. It should never re-initialize the cache."); + } + [TestMethod] - [Owner("flnarenj")] public async Task TestRidRefreshOnNotFoundAsync() { CosmosClient resourceClient = TestCommon.CreateCosmosClient(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/MockDocumentClient.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/MockDocumentClient.cs index 3f8242dfb2..631fd0c2dd 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/MockDocumentClient.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Mocks/MockDocumentClient.cs @@ -188,7 +188,6 @@ private void Init() It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), It.IsAny() ) ).Returns(Task.FromResult(routingMap)); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/PartitionKeyRangeHandlerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/PartitionKeyRangeHandlerTests.cs index b43cd35a93..8e8885cc73 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/PartitionKeyRangeHandlerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/PartitionKeyRangeHandlerTests.cs @@ -662,7 +662,7 @@ public async Task PartitionKeyRangeGoneTracePlumbingTest() new Mock().Object, new Mock().Object, collectionCache.Object); - partitionKeyRangeCache.Setup(c => c.TryLookupAsync(collectionRid, null, It.IsAny(), default, trace)) + partitionKeyRangeCache.Setup(c => c.TryLookupAsync(collectionRid, null, It.IsAny(), trace)) .ReturnsAsync(collectionRoutingMap); string collectionLink = "dbs/DvZRAA==/colls/DvZRAOvLgDM=/"; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Utils/MockDocumentClient.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Utils/MockDocumentClient.cs index 2d2fd70472..e678e6d53b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Utils/MockDocumentClient.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Utils/MockDocumentClient.cs @@ -232,7 +232,6 @@ private void Init() It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny(), It.IsAny() ) ).Returns(Task.FromResult(null));