From 1a5cf3f7886bb463f37e30a18e1e3124249f448d Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 7 Nov 2025 17:11:55 -0500 Subject: [PATCH 01/30] Enable 410-1022 on Head requests to bail out. --- .../directconnectivity/ConsistencyWriter.java | 105 ++++++++++++++++- .../directconnectivity/QuorumReader.java | 110 ++++++++++++++++-- .../directconnectivity/StoreReader.java | 36 +++--- 3 files changed, 219 insertions(+), 32 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java index 92a781cf071c..00300a56a5a2 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java @@ -17,6 +17,7 @@ import com.azure.cosmos.implementation.ISessionContainer; import com.azure.cosmos.implementation.Integers; import com.azure.cosmos.implementation.InternalServerErrorException; +import com.azure.cosmos.implementation.MutableVolatile; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.RMResources; import com.azure.cosmos.implementation.RequestChargeTracker; @@ -27,7 +28,6 @@ import com.azure.cosmos.implementation.Strings; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.apachecommons.collections.ComparatorUtils; -import com.azure.cosmos.implementation.directconnectivity.rntbd.ClosedClientTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.Exceptions; @@ -149,6 +149,9 @@ Mono writePrivateAsync( TimeoutHelper timeout, boolean forceRefresh) { + final MutableVolatile cosmosExceptionValueHolder = new MutableVolatile<>(null); + final MutableVolatile bailFromWriteBarrierLoopValueHolder = new MutableVolatile<>(false); + if (timeout.isElapsed() && // skip throwing RequestTimeout on first retry because the first retry with // force address refresh header can be critical to recover for example from @@ -280,7 +283,7 @@ Mono writePrivateAsync( false, primaryURI.get(), replicaStatusList); - return barrierForGlobalStrong(request, response); + return barrierForGlobalStrong(request, response, cosmosExceptionValueHolder, bailFromWriteBarrierLoopValueHolder); }) .doFinally(signalType -> { if (signalType != SignalType.CANCEL) { @@ -296,11 +299,16 @@ Mono writePrivateAsync( } else { Mono barrierRequestObs = BarrierRequestHelper.createAsync(this.diagnosticsClientContext, request, this.authorizationTokenProvider, null, request.requestContext.globalCommittedSelectedLSN); - return barrierRequestObs.flatMap(barrierRequest -> waitForWriteBarrierAsync(barrierRequest, request.requestContext.globalCommittedSelectedLSN) + return barrierRequestObs.flatMap(barrierRequest -> waitForWriteBarrierAsync(barrierRequest, request.requestContext.globalCommittedSelectedLSN, cosmosExceptionValueHolder, bailFromWriteBarrierLoopValueHolder) .flatMap(v -> { if (!v) { logger.info("ConsistencyWriter: Write barrier has not been met for global strong request. SelectedGlobalCommittedLsn: {}", request.requestContext.globalCommittedSelectedLSN); + + if (cosmosExceptionValueHolder.v != null) { + return Mono.error(cosmosExceptionValueHolder.v); + } + return Mono.error(new GoneException(RMResources.GlobalStrongWriteBarrierNotMet, HttpConstants.SubStatusCodes.GLOBAL_STRONG_WRITE_BARRIER_NOT_MET)); } @@ -324,7 +332,12 @@ boolean isGlobalStrongRequest(RxDocumentServiceRequest request, StoreResponse re return false; } - Mono barrierForGlobalStrong(RxDocumentServiceRequest request, StoreResponse response) { + Mono barrierForGlobalStrong( + RxDocumentServiceRequest request, + StoreResponse response, + MutableVolatile cosmosExceptionValueHolder, + MutableVolatile bailFromWriteBarrierLoopValueHolder) { + try { if (ReplicatedResourceClient.isGlobalStrongEnabled() && this.isGlobalStrongRequest(request, response)) { Utils.ValueHolder lsn = Utils.ValueHolder.initialize(-1L); @@ -355,12 +368,17 @@ Mono barrierForGlobalStrong(RxDocumentServiceRequest request, Sto request.requestContext.globalCommittedSelectedLSN); return barrierRequestObs.flatMap(barrierRequest -> { - Mono barrierWait = this.waitForWriteBarrierAsync(barrierRequest, request.requestContext.globalCommittedSelectedLSN); + Mono barrierWait = this.waitForWriteBarrierAsync(barrierRequest, request.requestContext.globalCommittedSelectedLSN, cosmosExceptionValueHolder, bailFromWriteBarrierLoopValueHolder); return barrierWait.flatMap(res -> { if (!res) { logger.error("ConsistencyWriter: Write barrier has not been met for global strong request. SelectedGlobalCommittedLsn: {}", request.requestContext.globalCommittedSelectedLSN); + + if (cosmosExceptionValueHolder.v != null) { + return Mono.error(cosmosExceptionValueHolder.v); + } + // RxJava1 doesn't allow throwing checked exception return Mono.error(new GoneException(RMResources.GlobalStrongWriteBarrierNotMet, HttpConstants.SubStatusCodes.GLOBAL_STRONG_WRITE_BARRIER_NOT_MET)); @@ -384,7 +402,12 @@ Mono barrierForGlobalStrong(RxDocumentServiceRequest request, Sto } } - private Mono waitForWriteBarrierAsync(RxDocumentServiceRequest barrierRequest, long selectedGlobalCommittedLsn) { + private Mono waitForWriteBarrierAsync( + RxDocumentServiceRequest barrierRequest, + long selectedGlobalCommittedLsn, + MutableVolatile cosmosExceptionValueHolder, + MutableVolatile bailFromWriteBarrierLoop) { + AtomicInteger writeBarrierRetryCount = new AtomicInteger(ConsistencyWriter.MAX_NUMBER_OF_WRITE_BARRIER_READ_RETRIES); AtomicLong maxGlobalCommittedLsnReceived = new AtomicLong(0); return Flux.defer(() -> { @@ -403,6 +426,27 @@ private Mono waitForWriteBarrierAsync(RxDocumentServiceRequest barrierR false /*forceReadAll*/); return storeResultListObs.flatMap( responses -> { + + boolean isAvoidQuorumSelectionStoreResult = false; + CosmosException cosmosExceptionFromStoreResult = null; + + for (StoreResult storeResult : responses) { + if (storeResult.isAvoidQuorumSelectionException) { + isAvoidQuorumSelectionStoreResult = true; + cosmosExceptionFromStoreResult = storeResult.getException(); + break; + } + } + + if (isAvoidQuorumSelectionStoreResult) { + return this.isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( + barrierRequest, + selectedGlobalCommittedLsn, + cosmosExceptionValueHolder, + bailFromWriteBarrierLoop, + cosmosExceptionFromStoreResult); + } + if (responses != null && responses.stream().anyMatch(response -> response.globalCommittedLSN >= selectedGlobalCommittedLsn)) { return Mono.just(Boolean.TRUE); } @@ -459,6 +503,55 @@ static void getLsnAndGlobalCommittedLsn(StoreResponse response, Utils.ValueHolde } } + private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( + RxDocumentServiceRequest barrierRequest, + long selectedGlobalCommittedLsn, + MutableVolatile cosmosExceptionValueHolder, + MutableVolatile bailFromWriteBarrierLoop, + CosmosException cosmosExceptionInStoreResult) { + + return performBarrierOnPrimary( + barrierRequest, + selectedGlobalCommittedLsn) + .flatMap(barrierStatusFromPrimary -> { + + if (barrierStatusFromPrimary) { + bailFromWriteBarrierLoop.v = true; + cosmosExceptionValueHolder.v = null; + + return Mono.just(true); + } + + bailFromWriteBarrierLoop.v = true; + cosmosExceptionValueHolder.v = Utils.createCosmosException( + HttpConstants.StatusCodes.REQUEST_TIMEOUT, + cosmosExceptionInStoreResult.getSubStatusCode(), + null, + null); + return Mono.just(false); + }); + } + + private Mono performBarrierOnPrimary( + RxDocumentServiceRequest entity, + long selectedGlobalCommittedLSN) { + + entity.requestContext.forceRefreshAddressCache = true; + Mono storeResultObs = this.storeReader.readPrimaryAsync( + entity, true, false /*useSessionToken*/); + + return storeResultObs.flatMap(storeResult -> { + if (!storeResult.isValid) { + return Mono.just(false); + } + + boolean hasRequiredGlobalCommittedLsn = + selectedGlobalCommittedLSN <= 0 || storeResult.globalCommittedLSN >= selectedGlobalCommittedLSN; + + return Mono.just(hasRequiredGlobalCommittedLsn); + }); + } + void startBackgroundAddressRefresh(RxDocumentServiceRequest request) { this.addressSelector.resolvePrimaryUriAsync(request, true) .publishOn(Schedulers.boundedElastic()) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java index 1f01e66076d6..00f42314a4e5 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java @@ -12,7 +12,6 @@ import com.azure.cosmos.implementation.Exceptions; import com.azure.cosmos.implementation.GoneException; import com.azure.cosmos.implementation.HttpConstants; -import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.InternalServerErrorException; import com.azure.cosmos.implementation.Configs; import com.azure.cosmos.implementation.IAuthorizationTokenProvider; @@ -22,6 +21,7 @@ import com.azure.cosmos.implementation.RMResources; import com.azure.cosmos.implementation.RequestChargeTracker; import com.azure.cosmos.implementation.RxDocumentServiceRequest; +import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.apachecommons.lang.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,6 +135,8 @@ public Mono readStrongAsync( final MutableVolatile shouldRetryOnSecondary = new MutableVolatile<>(false); final MutableVolatile hasPerformedReadFromPrimary = new MutableVolatile<>(false); final MutableVolatile includePrimary = new MutableVolatile<>(false); + final MutableVolatile cosmosExceptionValueHolder = new MutableVolatile<>(null); + final MutableVolatile bailOnBarrierValueHolder = new MutableVolatile<>(false); return Flux.defer( // the following will be repeated till the repeat().takeUntil(.) condition is satisfied. @@ -149,7 +151,9 @@ public Mono readStrongAsync( entity, readQuorumValue, includePrimary.v, - readMode); + readMode, + cosmosExceptionValueHolder, + bailOnBarrierValueHolder); return secondaryQuorumReadResultObs.flux().flatMap( secondaryQuorumReadResult -> { @@ -185,7 +189,9 @@ public Mono readStrongAsync( readQuorumValue, secondaryQuorumReadResult.selectedLsn, secondaryQuorumReadResult.globalCommittedSelectedLsn, - readMode); + readMode, + cosmosExceptionValueHolder, + bailOnBarrierValueHolder); return readBarrierObs.flux().flatMap( readBarrier -> { @@ -291,11 +297,15 @@ public Mono readStrongAsync( .switchIfEmpty(Flux.defer(() -> { logger.info("Could not complete read quorum with read quorum value of {}", readQuorumValue); - return Flux.error(new GoneException( + if (cosmosExceptionValueHolder.v != null) { + return Flux.error(cosmosExceptionValueHolder.v); + } + + return Flux.error(new GoneException( String.format( RMResources.ReadQuorumNotMet, readQuorumValue), - HttpConstants.SubStatusCodes.READ_QUORUM_NOT_MET)); + HttpConstants.SubStatusCodes.READ_QUORUM_NOT_MET)); })) .take(1) .single(); @@ -305,7 +315,10 @@ private Mono readQuorumAsync( RxDocumentServiceRequest entity, int readQuorum, boolean includePrimary, - ReadMode readMode) { + ReadMode readMode, + MutableVolatile cosmosExceptionValueHolder, + MutableVolatile bailOnBarrierValueHolder) { + if (entity.requestContext.timeoutHelper.isElapsed()) { return Mono.error(new GoneException()); } @@ -327,7 +340,7 @@ private Mono readQuorumAsync( Mono barrierRequestObs = BarrierRequestHelper.createAsync(this.diagnosticsClientContext, entity, this.authorizationTokenProvider, readLsn, globalCommittedLSN); return barrierRequestObs.flatMap( barrierRequest -> { - Mono waitForObs = this.waitForReadBarrierAsync(barrierRequest, false, readQuorum, readLsn, globalCommittedLSN, readMode); + Mono waitForObs = this.waitForReadBarrierAsync(barrierRequest, false, readQuorum, readLsn, globalCommittedLSN, readMode, cosmosExceptionValueHolder, bailOnBarrierValueHolder); return waitForObs.flatMap( waitFor -> { if (!waitFor) { @@ -616,7 +629,9 @@ private Mono waitForReadBarrierAsync( final int readQuorum, final long readBarrierLsn, final long targetGlobalCommittedLSN, - ReadMode readMode) { + ReadMode readMode, + MutableVolatile cosmosExceptionValueHolder, + MutableVolatile bailFromReadBarrierLoopValueHolder) { AtomicInteger readBarrierRetryCount = new AtomicInteger(maxNumberOfReadBarrierReadRetries); AtomicInteger readBarrierRetryCountMultiRegion = new AtomicInteger(maxBarrierRetriesForMultiRegion); @@ -635,6 +650,27 @@ private Mono waitForReadBarrierAsync( return responsesObs.flux().flatMap( responses -> { + boolean isAvoidQuorumSelectionStoreResult = false; + CosmosException cosmosExceptionFromStoreResult = null; + + for (StoreResult storeResult : responses) { + if (storeResult.isAvoidQuorumSelectionException) { + isAvoidQuorumSelectionStoreResult = true; + cosmosExceptionFromStoreResult = storeResult.getException(); + break; + } + } + + if (isAvoidQuorumSelectionStoreResult) { + return this.isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( + barrierRequest, + readBarrierLsn, + targetGlobalCommittedLSN, + cosmosExceptionValueHolder, + bailFromReadBarrierLoopValueHolder, + cosmosExceptionFromStoreResult); + } + long maxGlobalCommittedLsnInResponses = responses.size() > 0 ? responses.stream() .mapToLong(response -> response.globalCommittedLSN).max().getAsLong() : 0; @@ -677,6 +713,10 @@ private Mono waitForReadBarrierAsync( return Flux.just(true); } + if (bailFromReadBarrierLoopValueHolder.v) { + return Flux.just(false); + } + // we will go into global strong read barrier mode for global strong requests after regular barrier calls have been exhausted. if (targetGlobalCommittedLSN > 0) { return Flux.defer(() -> { @@ -823,6 +863,60 @@ private boolean isQuorumMet( return isQuorumMet; } + private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( + RxDocumentServiceRequest barrierRequest, + long readBarrierLsn, + long targetGlobalCommittedLSN, + MutableVolatile cosmosExceptionValueHolder, + MutableVolatile bailFromReadBarrierLoop, + CosmosException cosmosExceptionFromStoreResult) { + + return performBarrierOnPrimary( + barrierRequest, + true, + readBarrierLsn, + targetGlobalCommittedLSN).flatMap(barrierStatusFromPrimary -> { + + if (barrierStatusFromPrimary) { + bailFromReadBarrierLoop.v = true; + cosmosExceptionValueHolder.v = null; + + return Mono.just(true); + } + + bailFromReadBarrierLoop.v = true; + cosmosExceptionValueHolder.v = Utils.createCosmosException( + HttpConstants.StatusCodes.SERVICE_UNAVAILABLE, + cosmosExceptionFromStoreResult.getSubStatusCode(), + null, + null); + return Mono.just(false); + }); + } + + private Mono performBarrierOnPrimary( + RxDocumentServiceRequest entity, + boolean requiresValidLsn, + long readBarrierLsn, + long targetGlobalCommittedLSN) { + + entity.requestContext.forceRefreshAddressCache = true; + Mono storeResultObs = this.storeReader.readPrimaryAsync( + entity, requiresValidLsn, false /*useSessionToken*/); + + return storeResultObs.flatMap(storeResult -> { + if (!storeResult.isValid) { + return Mono.just(false); + } + + boolean hasRequiredLsn = storeResult.lsn >= readBarrierLsn; + boolean hasRequiredGlobalCommittedLsn = + targetGlobalCommittedLSN <= 0 || storeResult.globalCommittedLSN >= targetGlobalCommittedLSN; + + return Mono.just(hasRequiredLsn && hasRequiredGlobalCommittedLsn); + }); + } + private enum ReadQuorumResultKind { QuorumMet, QuorumSelected, diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java index 5169dfb121a7..5d7da614fd46 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java @@ -68,10 +68,10 @@ public StoreReader( public Mono> readMultipleReplicaAsync( RxDocumentServiceRequest entity, - boolean includePrimary, - int replicaCountToRead, - boolean requiresValidLsn, - boolean useSessionToken, + boolean includePrimary, // false + int replicaCountToRead, // 2 + boolean requiresValidLsn, // true + boolean useSessionToken, // false ReadMode readMode) { return readMultipleReplicaAsync(entity, includePrimary, replicaCountToRead, requiresValidLsn, useSessionToken, readMode, false, false); } @@ -90,13 +90,13 @@ public Mono> readMultipleReplicaAsync( */ public Mono> readMultipleReplicaAsync( RxDocumentServiceRequest entity, - boolean includePrimary, - int replicaCountToRead, - boolean requiresValidLsn, - boolean useSessionToken, + boolean includePrimary, // false + int replicaCountToRead, // 2 + boolean requiresValidLsn, // true + boolean useSessionToken, // false ReadMode readMode, - boolean checkMinLSN, - boolean forceReadAll) { + boolean checkMinLSN, // false + boolean forceReadAll /* false */) { if (entity.requestContext.timeoutHelper.isElapsed()) { return Mono.error(new GoneException()); @@ -283,7 +283,7 @@ private Flux> readFromReplicas(List resultCollect // todo: fail fast when barrier requests also hit isAvoidQuorumSelectionException? // todo: https://github.com/Azure/azure-sdk-for-java/issues/46135 - if (!entity.isBarrierRequest) { + // if (!entity.isBarrierRequest) { // isAvoidQuorumSelectionException is a special case where we want to enable the enclosing data plane operation // to fail fast in the region where a quorum selection is being attempted @@ -310,7 +310,7 @@ private Flux> readFromReplicas(List resultCollect // continue to the next store result continue; - } + //} } if (srr.isValid) { @@ -435,13 +435,13 @@ private ReadReplicaResult createReadReplicaResult(List responseResu * @return ReadReplicaResult which indicates the LSN and whether Quorum was Met / Not Met etc */ private Mono readMultipleReplicasInternalAsync(RxDocumentServiceRequest entity, - boolean includePrimary, - int replicaCountToRead, - boolean requiresValidLsn, - boolean useSessionToken, + boolean includePrimary, // false + int replicaCountToRead, // 2 + boolean requiresValidLsn, // true + boolean useSessionToken, // false ReadMode readMode, - boolean checkMinLSN, - boolean forceReadAll) { + boolean checkMinLSN, // false + boolean forceReadAll /* false */) { if (entity.requestContext.timeoutHelper.isElapsed()) { return Mono.error(new GoneException()); } From f571a5ef80bedfc4d174c7ff754da11c78888a6e Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 10 Nov 2025 14:34:39 -0500 Subject: [PATCH 02/30] Bail fast on barrier on reads. --- .../cosmos/ExitFromConsistencyLayerTests.java | 313 ++++++++++++++++++ ...ortClientWithStoreResponseInterceptor.java | 72 ++++ .../cosmos/StoreResponseInterceptorUtils.java | 53 +++ .../directconnectivity/ReflectionUtils.java | 4 + .../directconnectivity/ConsistencyWriter.java | 21 +- .../directconnectivity/QuorumReader.java | 49 ++- .../RntbdTransportClient.java | 12 + .../directconnectivity/StoreReader.java | 32 +- .../directconnectivity/StoreResponse.java | 16 + 9 files changed, 532 insertions(+), 40 deletions(-) create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RntbdTransportClientWithStoreResponseInterceptor.java create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java new file mode 100644 index 000000000000..92240203a7e0 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -0,0 +1,313 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos; + +import com.azure.cosmos.implementation.AsyncDocumentClient; +import com.azure.cosmos.implementation.DatabaseAccount; +import com.azure.cosmos.implementation.DatabaseAccountLocation; +import com.azure.cosmos.implementation.GlobalEndpointManager; +import com.azure.cosmos.implementation.HttpConstants; +import com.azure.cosmos.implementation.OperationType; +import com.azure.cosmos.implementation.RxDocumentClientImpl; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.directconnectivity.ConsistencyReader; +import com.azure.cosmos.implementation.directconnectivity.ConsistencyWriter; +import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; +import com.azure.cosmos.implementation.directconnectivity.ReplicatedResourceClient; +import com.azure.cosmos.implementation.directconnectivity.RntbdTransportClient; +import com.azure.cosmos.implementation.directconnectivity.StoreClient; +import com.azure.cosmos.implementation.directconnectivity.StoreReader; +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.rx.TestSuiteBase; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +public class ExitFromConsistencyLayerTests extends TestSuiteBase { + + private CosmosAsyncClient client; + private volatile CosmosAsyncDatabase database; + private volatile CosmosAsyncContainer container; + private List preferredRegions; + private Map regionNameToEndpoint; + + @BeforeClass(groups = {"multi-region"}) + public void beforeClass() { + + try (CosmosAsyncClient dummy = getClientBuilder().buildAsyncClient()) { + AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(dummy); + GlobalEndpointManager globalEndpointManager = asyncDocumentClient.getGlobalEndpointManager(); + + DatabaseAccount databaseAccount = globalEndpointManager.getLatestDatabaseAccount(); + + AccountLevelLocationContext accountLevelContext = getAccountLevelLocationContext(databaseAccount, false); + + // Set preferred regions starting with secondary region + this.preferredRegions = new ArrayList<>(accountLevelContext.serviceOrderedReadableRegions); + if (this.preferredRegions.size() > 1) { + // Swap first and second to make secondary region preferred + String temp = this.preferredRegions.get(0); + this.preferredRegions.set(0, this.preferredRegions.get(1)); + this.preferredRegions.set(1, temp); + } + + this.regionNameToEndpoint = accountLevelContext.regionNameToEndpoint; + this.database = getSharedCosmosDatabase(dummy); + this.container = getSharedSinglePartitionCosmosContainer(dummy); + } + } + + @Factory(dataProvider = "clientBuildersWithDirectTcp") + public ExitFromConsistencyLayerTests(CosmosClientBuilder clientBuilder) { + super(clientBuilder); + } + + @DataProvider(name = "headFailureScenarios") + public static Object[][] headFailureScenarios() { + return new Object[][]{ + // headFailureCount, operationType + { 1, OperationType.Create }, + { 2, OperationType.Create }, + { 3, OperationType.Create }, + { 4, OperationType.Create }, + { 1, OperationType.Read }, + { 2, OperationType.Read }, + { 4, OperationType.Read }, + }; + } + + @Test(groups = {"multi-region"}, dataProvider = "headFailureScenarios" /*, timeOut = 2 * TIMEOUT*/) + public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, OperationType operationTypeForWhichBarrierFlowIsTriggered) throws Exception { + + CosmosAsyncClient targetClient = getClientBuilder() + .preferredRegions(this.preferredRegions) + .directMode() + .consistencyLevel(ConsistencyLevel.STRONG) + .buildAsyncClient(); + + AtomicInteger failureCountTracker = new AtomicInteger(); + Utils.ValueHolder originalRntbdTransportClientHolder = new Utils.ValueHolder<>(); + + RntbdTransportClientWithStoreResponseInterceptor interceptorClient = createClientWithInterceptor(targetClient, originalRntbdTransportClientHolder); + + try { + + // Setup test data + TestObject testObject = TestObject.create(); + + if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Create) { + interceptorClient + .setResponseInterceptor( + StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( + operationTypeForWhichBarrierFlowIsTriggered, + this.regionNameToEndpoint.get(this.preferredRegions.get(1)), + headFailureCount, + failureCountTracker, + HttpConstants.StatusCodes.GONE, + HttpConstants.SubStatusCodes.LEASE_NOT_FOUND + )); + } else if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Read) { + interceptorClient + .setResponseInterceptor( + StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( + operationTypeForWhichBarrierFlowIsTriggered, + this.regionNameToEndpoint.get(this.preferredRegions.get(0)), + headFailureCount, + failureCountTracker, + HttpConstants.StatusCodes.GONE, + HttpConstants.SubStatusCodes.LEASE_NOT_FOUND + )); + } + + try { + CosmosAsyncDatabase targetAsyncDatabase = targetClient.getDatabase(this.database.getId()); + CosmosAsyncContainer targetContainer = targetAsyncDatabase.getContainer(this.container.getId()); + + Thread.sleep(5000); // Wait for collection to be available to be read + + // Assert based on operation type and failure count + if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Create) { + + // Perform the operation + CosmosItemResponse response = targetContainer.createItem(testObject).block(); + + // For Create, Head can only fail up to 2 times + if (headFailureCount <= 1) { + // Should eventually succeed + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.CREATED); + + // Check diagnostics - should have contacted only one region for create + CosmosDiagnostics diagnostics = response.getDiagnostics(); + assertThat(diagnostics).isNotNull(); + + validateContactedRegions(diagnostics, 1); + + } else { + // Should timeout with 408 + fail("Should have thrown timeout exception"); + } + } else { + + targetContainer.createItem(testObject).block(); + + CosmosItemResponse response + = targetContainer.readItem(testObject.getId(), new PartitionKey(testObject.getMypk()), TestObject.class).block(); + + if (headFailureCount <= 2) { + // Should eventually succeed + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.OK); + + // Check diagnostics - should have contacted only one region for create + CosmosDiagnostics diagnostics = response.getDiagnostics(); + assertThat(diagnostics).isNotNull(); + + validateContactedRegions(diagnostics, 1); + } else { + // Should eventually succeed + assertThat(response).isNotNull(); + assertThat(response.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.OK); + + // Check diagnostics - should have contacted only one region for create + CosmosDiagnostics diagnostics = response.getDiagnostics(); + assertThat(diagnostics).isNotNull(); + + validateContactedRegions(diagnostics, 2); + } + } + + } catch (CosmosException e) { + // Expected for some scenarios + if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Create) { + + if (headFailureCount <= 1) { + fail("Should have succeeded for create with head failures less than or equal to 2"); + } else { + // Should get 408 timeout + assertThat(e.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.REQUEST_TIMEOUT); + + CosmosDiagnostics diagnostics = e.getDiagnostics(); + + validateContactedRegions(diagnostics, 1); + } + } + + if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Read) { + fail("Read operation should have succeeded even with head failures"); + } + } + + } finally { + + if (originalRntbdTransportClientHolder.v != null) { + originalRntbdTransportClientHolder.v.close(); + } + + interceptorClient.close(); + safeClose(targetClient); + } + } + + private RntbdTransportClientWithStoreResponseInterceptor createClientWithInterceptor( + CosmosAsyncClient targetClient, + Utils.ValueHolder originalRntbdTransportClientHolder) { + + // Get internal client + AsyncDocumentClient asyncDocumentClient = ReflectionUtils.getAsyncDocumentClient(targetClient); + RxDocumentClientImpl rxDocumentClient = (RxDocumentClientImpl) asyncDocumentClient; + + // Get store client and components + StoreClient storeClient = ReflectionUtils.getStoreClient(rxDocumentClient); + ReplicatedResourceClient replicatedResourceClient = ReflectionUtils.getReplicatedResourceClient(storeClient); + ConsistencyReader consistencyReader = ReflectionUtils.getConsistencyReader(replicatedResourceClient); + ConsistencyWriter consistencyWriter = ReflectionUtils.getConsistencyWriter(replicatedResourceClient); + StoreReader storeReaderFromConsistencyReader = ReflectionUtils.getStoreReader(consistencyReader); + StoreReader storeReaderFromConsistencyWriter = ReflectionUtils.getStoreReader(consistencyWriter); + + // Get the original transport client + RntbdTransportClient originalTransportClient = (RntbdTransportClient) ReflectionUtils.getTransportClient(replicatedResourceClient); + originalRntbdTransportClientHolder.v = originalTransportClient; + + // Create interceptor client + RntbdTransportClientWithStoreResponseInterceptor interceptorClient = + new RntbdTransportClientWithStoreResponseInterceptor(originalTransportClient); + + // Set the interceptor client on both reader and writer + ReflectionUtils.setTransportClient(storeReaderFromConsistencyReader, interceptorClient); + ReflectionUtils.setTransportClient(storeReaderFromConsistencyWriter, interceptorClient); + ReflectionUtils.setTransportClient(consistencyWriter, interceptorClient); + + return interceptorClient; + } + + private void validateContactedRegions(CosmosDiagnostics diagnostics, int expectedRegionsContactedCount) { + + CosmosDiagnosticsContext cosmosDiagnosticsContext = diagnostics.getDiagnosticsContext(); + + assertThat(cosmosDiagnosticsContext).isNotNull(); + assertThat(cosmosDiagnosticsContext.getContactedRegionNames()).isNotNull(); + assertThat(cosmosDiagnosticsContext.getContactedRegionNames()).isNotEmpty(); + assertThat(cosmosDiagnosticsContext.getContactedRegionNames().size()).isEqualTo(expectedRegionsContactedCount); + } + + private AccountLevelLocationContext getAccountLevelLocationContext(DatabaseAccount databaseAccount, boolean writeOnly) { + Iterator locationIterator = + writeOnly ? databaseAccount.getWritableLocations().iterator() : databaseAccount.getReadableLocations().iterator(); + + List serviceOrderedReadableRegions = new ArrayList<>(); + List serviceOrderedWriteableRegions = new ArrayList<>(); + Map regionMap = new ConcurrentHashMap<>(); + + while (locationIterator.hasNext()) { + DatabaseAccountLocation accountLocation = locationIterator.next(); + regionMap.put(accountLocation.getName(), accountLocation.getEndpoint()); + + if (writeOnly) { + serviceOrderedWriteableRegions.add(accountLocation.getName()); + } else { + serviceOrderedReadableRegions.add(accountLocation.getName()); + } + } + + return new AccountLevelLocationContext( + serviceOrderedReadableRegions, + serviceOrderedWriteableRegions, + regionMap); + } + + private static class AccountLevelLocationContext { + private final List serviceOrderedReadableRegions; + private final List serviceOrderedWriteableRegions; + private final Map regionNameToEndpoint; + + public AccountLevelLocationContext( + List serviceOrderedReadableRegions, + List serviceOrderedWriteableRegions, + Map regionNameToEndpoint) { + + this.serviceOrderedReadableRegions = serviceOrderedReadableRegions; + this.serviceOrderedWriteableRegions = serviceOrderedWriteableRegions; + this.regionNameToEndpoint = regionNameToEndpoint; + } + } + + @AfterClass(groups = {"multi-region"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + public void afterClass() { + safeClose(client); + } +} diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RntbdTransportClientWithStoreResponseInterceptor.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RntbdTransportClientWithStoreResponseInterceptor.java new file mode 100644 index 000000000000..5e64b209d189 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RntbdTransportClientWithStoreResponseInterceptor.java @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos; + +import com.azure.cosmos.implementation.GlobalEndpointManager; +import com.azure.cosmos.implementation.RxDocumentServiceRequest; +import com.azure.cosmos.implementation.directconnectivity.RntbdTransportClient; +import com.azure.cosmos.implementation.directconnectivity.StoreResponse; +import com.azure.cosmos.implementation.directconnectivity.Uri; +import com.azure.cosmos.implementation.directconnectivity.rntbd.ProactiveOpenConnectionsProcessor; +import com.azure.cosmos.implementation.faultinjection.IFaultInjectorProvider; +import com.azure.cosmos.models.CosmosContainerIdentity; +import reactor.core.publisher.Mono; + +import java.util.List; +import java.util.function.BiFunction; + +public class RntbdTransportClientWithStoreResponseInterceptor extends RntbdTransportClient { + private final RntbdTransportClient underlying; + private BiFunction responseInterceptor; + + public RntbdTransportClientWithStoreResponseInterceptor(RntbdTransportClient underlying) { + super(underlying); + this.underlying = underlying; + } + + public void setResponseInterceptor(BiFunction responseInterceptor) { + this.responseInterceptor = responseInterceptor; + } + + @Override + public Mono invokeStoreAsync(Uri physicalAddress, RxDocumentServiceRequest request) { + return this.underlying.invokeStoreAsync(physicalAddress, request) + .map(response -> { + if (responseInterceptor != null) { + return responseInterceptor.apply(request, response); + } + return response; + }); + } + + @Override + public void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvider) { + this.underlying.configureFaultInjectorProvider(injectorProvider); + } + + @Override + public GlobalEndpointManager getGlobalEndpointManager() { + return this.underlying.getGlobalEndpointManager(); + } + + @Override + public ProactiveOpenConnectionsProcessor getProactiveOpenConnectionsProcessor() { + return this.underlying.getProactiveOpenConnectionsProcessor(); + } + + @Override + public void recordOpenConnectionsAndInitCachesCompleted(List cosmosContainerIdentities) { + this.underlying.recordOpenConnectionsAndInitCachesCompleted(cosmosContainerIdentities); + } + + @Override + public void recordOpenConnectionsAndInitCachesStarted(List cosmosContainerIdentities) { + this.underlying.recordOpenConnectionsAndInitCachesStarted(cosmosContainerIdentities); + } + + @Override + public void close() { + this.underlying.close(); + } +} diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java new file mode 100644 index 000000000000..722d17d26900 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos; + +import com.azure.cosmos.implementation.OperationType; +import com.azure.cosmos.implementation.RxDocumentServiceRequest; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.directconnectivity.StoreResponse; +import com.azure.cosmos.implementation.directconnectivity.WFConstants; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; + +public class StoreResponseInterceptorUtils { + + public static BiFunction forceBarrierFollowedByBarrierFailure( + OperationType operationType, + String regionName, + int maxAllowedFailureCount, + AtomicInteger failureCount, + int statusCode, + int subStatusCode) { + + return (request, storeResponse) -> { + + if (OperationType.Create.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { + + long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); + long manipulatedGCLSN = localLsn - 1; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + } + + if (OperationType.Read.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { + + long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); + long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); + long manipulatedGCLSN = Math.min(localLsn, itemLsn) - 1; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + } + + if (OperationType.Head.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { + if (failureCount.incrementAndGet() <= maxAllowedFailureCount) { + throw Utils.createCosmosException(statusCode, subStatusCode, new Exception("An intercepted exception occurred. Check status and substatus code for details."), null); + } + } + + return storeResponse; + }; + } +} diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java index cf126f74b3fe..672daffc5c27 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/ReflectionUtils.java @@ -309,6 +309,10 @@ public static StoreReader getStoreReader(ConsistencyReader consistencyReader) { return get(StoreReader.class, consistencyReader, "storeReader"); } + public static StoreReader getStoreReader(ConsistencyWriter consistencyWriter) { + return get(StoreReader.class, consistencyWriter, "storeReader"); + } + public static void setStoreReader(ConsistencyReader consistencyReader, StoreReader storeReader) { set(consistencyReader, storeReader, "storeReader"); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java index 00300a56a5a2..c4f5ee117caf 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java @@ -510,7 +510,7 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep MutableVolatile bailFromWriteBarrierLoop, CosmosException cosmosExceptionInStoreResult) { - return performBarrierOnPrimary( + return performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( barrierRequest, selectedGlobalCommittedLsn) .flatMap(barrierStatusFromPrimary -> { @@ -526,13 +526,13 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep cosmosExceptionValueHolder.v = Utils.createCosmosException( HttpConstants.StatusCodes.REQUEST_TIMEOUT, cosmosExceptionInStoreResult.getSubStatusCode(), - null, + cosmosExceptionInStoreResult, null); return Mono.just(false); }); } - private Mono performBarrierOnPrimary( + private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( RxDocumentServiceRequest entity, long selectedGlobalCommittedLSN) { @@ -541,15 +541,16 @@ private Mono performBarrierOnPrimary( entity, true, false /*useSessionToken*/); return storeResultObs.flatMap(storeResult -> { - if (!storeResult.isValid) { - return Mono.just(false); - } + if (!storeResult.isValid) { + return Mono.just(false); + } - boolean hasRequiredGlobalCommittedLsn = - selectedGlobalCommittedLSN <= 0 || storeResult.globalCommittedLSN >= selectedGlobalCommittedLSN; + boolean hasRequiredGlobalCommittedLsn = + selectedGlobalCommittedLSN <= 0 || storeResult.globalCommittedLSN >= selectedGlobalCommittedLSN; - return Mono.just(hasRequiredGlobalCommittedLsn); - }); + return Mono.just(hasRequiredGlobalCommittedLsn); + }) + .onErrorResume(throwable -> Mono.just(false)); } void startBackgroundAddressRefresh(RxDocumentServiceRequest request) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java index 00f42314a4e5..b04eccb6c180 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java @@ -137,6 +137,7 @@ public Mono readStrongAsync( final MutableVolatile includePrimary = new MutableVolatile<>(false); final MutableVolatile cosmosExceptionValueHolder = new MutableVolatile<>(null); final MutableVolatile bailOnBarrierValueHolder = new MutableVolatile<>(false); + final AtomicInteger iterations = new AtomicInteger(0); return Flux.defer( // the following will be repeated till the repeat().takeUntil(.) condition is satisfied. @@ -191,7 +192,8 @@ public Mono readStrongAsync( secondaryQuorumReadResult.globalCommittedSelectedLsn, readMode, cosmosExceptionValueHolder, - bailOnBarrierValueHolder); + bailOnBarrierValueHolder, + iterations); return readBarrierObs.flux().flatMap( readBarrier -> { @@ -340,9 +342,21 @@ private Mono readQuorumAsync( Mono barrierRequestObs = BarrierRequestHelper.createAsync(this.diagnosticsClientContext, entity, this.authorizationTokenProvider, readLsn, globalCommittedLSN); return barrierRequestObs.flatMap( barrierRequest -> { - Mono waitForObs = this.waitForReadBarrierAsync(barrierRequest, false, readQuorum, readLsn, globalCommittedLSN, readMode, cosmosExceptionValueHolder, bailOnBarrierValueHolder); + Mono waitForObs = this.waitForReadBarrierAsync(barrierRequest, false, readQuorum, readLsn, globalCommittedLSN, readMode, cosmosExceptionValueHolder, bailOnBarrierValueHolder, new AtomicInteger(0)); return waitForObs.flatMap( waitFor -> { + + if (bailOnBarrierValueHolder.v && cosmosExceptionValueHolder.v != null) { + return Mono.just(new ReadQuorumResult( + entity.requestContext.requestChargeTracker, + ReadQuorumResultKind.QuorumNotPossibleInCurrentRegion, + readLsn, + globalCommittedLSN, + storeResult, + storeResponses, + cosmosExceptionValueHolder.v)); + } + if (!waitFor) { return Mono.just(new ReadQuorumResult( entity.requestContext.requestChargeTracker, @@ -631,14 +645,20 @@ private Mono waitForReadBarrierAsync( final long targetGlobalCommittedLSN, ReadMode readMode, MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailFromReadBarrierLoopValueHolder) { + MutableVolatile bailFromReadBarrierLoopValueHolder, + AtomicInteger iterations) { AtomicInteger readBarrierRetryCount = new AtomicInteger(maxNumberOfReadBarrierReadRetries); AtomicInteger readBarrierRetryCountMultiRegion = new AtomicInteger(maxBarrierRetriesForMultiRegion); AtomicLong maxGlobalCommittedLsn = new AtomicLong(0); + AtomicLong repetitions = new AtomicLong(0); return Flux.defer(() -> { + iterations.incrementAndGet(); + + logger.warn("Iteration Count : {}", iterations.get()); + if (barrierRequest.requestContext.timeoutHelper.isElapsed()) { return Flux.error(new GoneException()); } @@ -871,7 +891,7 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep MutableVolatile bailFromReadBarrierLoop, CosmosException cosmosExceptionFromStoreResult) { - return performBarrierOnPrimary( + return performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( barrierRequest, true, readBarrierLsn, @@ -888,13 +908,13 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep cosmosExceptionValueHolder.v = Utils.createCosmosException( HttpConstants.StatusCodes.SERVICE_UNAVAILABLE, cosmosExceptionFromStoreResult.getSubStatusCode(), - null, + cosmosExceptionFromStoreResult, null); return Mono.just(false); }); } - private Mono performBarrierOnPrimary( + private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( RxDocumentServiceRequest entity, boolean requiresValidLsn, long readBarrierLsn, @@ -905,16 +925,17 @@ private Mono performBarrierOnPrimary( entity, requiresValidLsn, false /*useSessionToken*/); return storeResultObs.flatMap(storeResult -> { - if (!storeResult.isValid) { - return Mono.just(false); - } + if (!storeResult.isValid) { + return Mono.just(false); + } - boolean hasRequiredLsn = storeResult.lsn >= readBarrierLsn; - boolean hasRequiredGlobalCommittedLsn = - targetGlobalCommittedLSN <= 0 || storeResult.globalCommittedLSN >= targetGlobalCommittedLSN; + boolean hasRequiredLsn = storeResult.lsn >= readBarrierLsn; + boolean hasRequiredGlobalCommittedLsn = + targetGlobalCommittedLSN <= 0 || storeResult.globalCommittedLSN >= targetGlobalCommittedLSN; - return Mono.just(hasRequiredLsn && hasRequiredGlobalCommittedLsn); - }); + return Mono.just(hasRequiredLsn && hasRequiredGlobalCommittedLsn); + }) + .onErrorResume(throwable -> Mono.just(false)); } private enum ReadQuorumResultKind { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java index 22c5ed8624b6..ac8f33cccb18 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java @@ -145,6 +145,18 @@ public RntbdTransportClient( globalEndpointManager); } + public RntbdTransportClient(RntbdTransportClient other) { + this.serverErrorInjector = other.serverErrorInjector; + this.endpointProvider = other.endpointProvider; + this.id = instanceCount.incrementAndGet(); + this.tag = RntbdTransportClient.tag(this.id); + this.channelAcquisitionContextLatencyThresholdInMillis = other.channelAcquisitionContextLatencyThresholdInMillis; + this.globalEndpointManager = other.globalEndpointManager; + this.addressSelector = other.addressSelector; + this.proactiveOpenConnectionsProcessor = other.proactiveOpenConnectionsProcessor; + this.metricConfig = other.metricConfig; + } + RntbdTransportClient( final Options options, final SslContext sslContext, diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java index 5d7da614fd46..fd6fb4b6fc85 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java @@ -68,10 +68,10 @@ public StoreReader( public Mono> readMultipleReplicaAsync( RxDocumentServiceRequest entity, - boolean includePrimary, // false - int replicaCountToRead, // 2 - boolean requiresValidLsn, // true - boolean useSessionToken, // false + boolean includePrimary, + int replicaCountToRead, + boolean requiresValidLsn, + boolean useSessionToken, ReadMode readMode) { return readMultipleReplicaAsync(entity, includePrimary, replicaCountToRead, requiresValidLsn, useSessionToken, readMode, false, false); } @@ -90,13 +90,13 @@ public Mono> readMultipleReplicaAsync( */ public Mono> readMultipleReplicaAsync( RxDocumentServiceRequest entity, - boolean includePrimary, // false - int replicaCountToRead, // 2 - boolean requiresValidLsn, // true - boolean useSessionToken, // false + boolean includePrimary, + int replicaCountToRead, + boolean requiresValidLsn, + boolean useSessionToken, ReadMode readMode, - boolean checkMinLSN, // false - boolean forceReadAll /* false */) { + boolean checkMinLSN, + boolean forceReadAll) { if (entity.requestContext.timeoutHelper.isElapsed()) { return Mono.error(new GoneException()); @@ -435,13 +435,13 @@ private ReadReplicaResult createReadReplicaResult(List responseResu * @return ReadReplicaResult which indicates the LSN and whether Quorum was Met / Not Met etc */ private Mono readMultipleReplicasInternalAsync(RxDocumentServiceRequest entity, - boolean includePrimary, // false - int replicaCountToRead, // 2 - boolean requiresValidLsn, // true - boolean useSessionToken, // false + boolean includePrimary, + int replicaCountToRead, + boolean requiresValidLsn, + boolean useSessionToken, ReadMode readMode, - boolean checkMinLSN, // false - boolean forceReadAll /* false */) { + boolean checkMinLSN, + boolean forceReadAll) { if (entity.requestContext.timeoutHelper.isElapsed()) { return Mono.error(new GoneException()); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java index c8539f09cba5..fe7df2a0168e 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java @@ -196,6 +196,22 @@ public String getHeaderValue(String attribute) { return null; } + public String setHeaderValue(String attribute, String value) { + if (this.responseHeaderValues == null || this.responseHeaderNames.length != this.responseHeaderValues.length) { + return null; + } + + for (int i = 0; i < responseHeaderNames.length; i++) { + if (responseHeaderNames[i].equalsIgnoreCase(attribute)) { + String oldValue = responseHeaderValues[i]; + responseHeaderValues[i] = value; + return oldValue; + } + } + + return null; + } + public double getRequestCharge() { String value = this.getHeaderValue(HttpConstants.HttpHeaders.REQUEST_CHARGE); if (StringUtils.isEmpty(value)) { From cf65b2020fc4a2374d33d5f8458aa9118cd0ebc0 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 11 Nov 2025 14:28:47 -0500 Subject: [PATCH 03/30] Enhance tests to ensure primary is contacted and barrier request count doesn't exceed a certain threshold. --- .../cosmos/ExitFromConsistencyLayerTests.java | 92 +++++++++++++++++-- 1 file changed, 84 insertions(+), 8 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index 92240203a7e0..653c25cd2e1f 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -4,10 +4,12 @@ package com.azure.cosmos; import com.azure.cosmos.implementation.AsyncDocumentClient; +import com.azure.cosmos.implementation.ClientSideRequestStatistics; import com.azure.cosmos.implementation.DatabaseAccount; import com.azure.cosmos.implementation.DatabaseAccountLocation; import com.azure.cosmos.implementation.GlobalEndpointManager; import com.azure.cosmos.implementation.HttpConstants; +import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.Utils; @@ -21,6 +23,7 @@ import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; import com.azure.cosmos.rx.TestSuiteBase; +import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; @@ -28,6 +31,7 @@ import org.testng.annotations.Test; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -39,7 +43,9 @@ public class ExitFromConsistencyLayerTests extends TestSuiteBase { - private CosmosAsyncClient client; + private static final ImplementationBridgeHelpers.CosmosAsyncClientHelper.CosmosAsyncClientAccessor cosmosAsyncClientAccessor + = ImplementationBridgeHelpers.CosmosAsyncClientHelper.getCosmosAsyncClientAccessor(); + private volatile CosmosAsyncDatabase database; private volatile CosmosAsyncContainer container; private List preferredRegions; @@ -90,15 +96,34 @@ public static Object[][] headFailureScenarios() { }; } - @Test(groups = {"multi-region"}, dataProvider = "headFailureScenarios" /*, timeOut = 2 * TIMEOUT*/) + @Test(groups = {"multi-region"}, dataProvider = "headFailureScenarios" , timeOut = 2 * TIMEOUT) public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, OperationType operationTypeForWhichBarrierFlowIsTriggered) throws Exception { CosmosAsyncClient targetClient = getClientBuilder() .preferredRegions(this.preferredRegions) - .directMode() - .consistencyLevel(ConsistencyLevel.STRONG) .buildAsyncClient(); + ConsistencyLevel effectiveConsistencyLevel + = cosmosAsyncClientAccessor.getEffectiveConsistencyLevel(targetClient, operationTypeForWhichBarrierFlowIsTriggered, null); + + ConnectionMode connectionModeOfClientUnderTest + = ConnectionMode.valueOf( + cosmosAsyncClientAccessor.getConnectionMode( + targetClient).toUpperCase()); + + if (!shouldTestExecutionHappen( + effectiveConsistencyLevel, + ConsistencyLevel.STRONG, + connectionModeOfClientUnderTest, + ConnectionMode.DIRECT)) { + + safeClose(targetClient); + throw new SkipException("Skipping test for arguments: " + + " OperationType: " + operationTypeForWhichBarrierFlowIsTriggered + + " ConsistencyLevel: " + effectiveConsistencyLevel + + " ConnectionMode: " + connectionModeOfClientUnderTest); + } + AtomicInteger failureCountTracker = new AtomicInteger(); Utils.ValueHolder originalRntbdTransportClientHolder = new Utils.ValueHolder<>(); @@ -156,7 +181,7 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation assertThat(diagnostics).isNotNull(); validateContactedRegions(diagnostics, 1); - + validateHeadRequestsInCosmosDiagnostics(diagnostics, 2); } else { // Should timeout with 408 fail("Should have thrown timeout exception"); @@ -178,6 +203,7 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation assertThat(diagnostics).isNotNull(); validateContactedRegions(diagnostics, 1); + validateHeadRequestsInCosmosDiagnostics(diagnostics, 4); } else { // Should eventually succeed assertThat(response).isNotNull(); @@ -188,6 +214,7 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation assertThat(diagnostics).isNotNull(); validateContactedRegions(diagnostics, 2); + validateHeadRequestsInCosmosDiagnostics(diagnostics, 4); } } @@ -204,6 +231,7 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation CosmosDiagnostics diagnostics = e.getDiagnostics(); validateContactedRegions(diagnostics, 1); + validateHeadRequestsInCosmosDiagnostics(diagnostics, 2); } } @@ -265,6 +293,39 @@ private void validateContactedRegions(CosmosDiagnostics diagnostics, int expecte assertThat(cosmosDiagnosticsContext.getContactedRegionNames().size()).isEqualTo(expectedRegionsContactedCount); } + private void validateHeadRequestsInCosmosDiagnostics( + CosmosDiagnostics diagnostics, + int expectedHeadRequestCount) { + + int headRequestCount = 0; + boolean primaryReplicaContacted = false; + + Collection clientSideRequestStatisticsCollection + = diagnostics.getClientSideRequestStatistics(); + + for (ClientSideRequestStatistics clientSideRequestStatistics : clientSideRequestStatisticsCollection) { + Collection storeResponseDiagnosticsList + = clientSideRequestStatistics.getSupplementalResponseStatisticsList(); + + for (ClientSideRequestStatistics.StoreResponseStatistics storeResponseStatistics : storeResponseDiagnosticsList) { + if (storeResponseStatistics.getRequestOperationType() == OperationType.Head) { + String storePhysicalAddressContacted + = storeResponseStatistics.getStoreResult().getStorePhysicalAddressAsString(); + + if (isPrimaryReplicaEndpoint(storePhysicalAddressContacted)) { + primaryReplicaContacted = true; + } + + headRequestCount++; + } + } + } + + assertThat(primaryReplicaContacted).isTrue(); + assertThat(headRequestCount).isGreaterThan(0); + assertThat(headRequestCount).isLessThanOrEqualTo(expectedHeadRequestCount); + } + private AccountLevelLocationContext getAccountLevelLocationContext(DatabaseAccount databaseAccount, boolean writeOnly) { Iterator locationIterator = writeOnly ? databaseAccount.getWritableLocations().iterator() : databaseAccount.getReadableLocations().iterator(); @@ -290,6 +351,23 @@ private AccountLevelLocationContext getAccountLevelLocationContext(DatabaseAccou regionMap); } + private boolean shouldTestExecutionHappen( + ConsistencyLevel accountConsistencyLevel, + ConsistencyLevel minimumConsistencyLevel, + ConnectionMode connectionModeOfClientUnderTest, + ConnectionMode expectedConnectionMode) { + + if (!connectionModeOfClientUnderTest.name().equalsIgnoreCase(expectedConnectionMode.name())) { + return false; + } + + return accountConsistencyLevel.equals(minimumConsistencyLevel); + } + + private static boolean isPrimaryReplicaEndpoint(String storePhysicalAddress) { + return storePhysicalAddress.endsWith("p/"); + } + private static class AccountLevelLocationContext { private final List serviceOrderedReadableRegions; private final List serviceOrderedWriteableRegions; @@ -307,7 +385,5 @@ public AccountLevelLocationContext( } @AfterClass(groups = {"multi-region"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) - public void afterClass() { - safeClose(client); - } + public void afterClass() {} } From 617c96161fa100d81492311db119ca5d42bb8307 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 11 Nov 2025 20:35:02 -0500 Subject: [PATCH 04/30] Enhance tests to ensure barrier post the QuorumSelected phase is invoked. --- .../cosmos/ExitFromConsistencyLayerTests.java | 169 +++++++++++++----- .../cosmos/StoreResponseInterceptorUtils.java | 49 ++++- .../directconnectivity/QuorumReader.java | 42 +++-- 3 files changed, 200 insertions(+), 60 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index 653c25cd2e1f..e772ce4e3e07 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -20,6 +20,8 @@ import com.azure.cosmos.implementation.directconnectivity.RntbdTransportClient; import com.azure.cosmos.implementation.directconnectivity.StoreClient; import com.azure.cosmos.implementation.directconnectivity.StoreReader; +import com.azure.cosmos.implementation.directconnectivity.StoreResponseDiagnostics; +import com.azure.cosmos.implementation.directconnectivity.StoreResultDiagnostics; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; import com.azure.cosmos.rx.TestSuiteBase; @@ -85,19 +87,32 @@ public ExitFromConsistencyLayerTests(CosmosClientBuilder clientBuilder) { @DataProvider(name = "headFailureScenarios") public static Object[][] headFailureScenarios() { return new Object[][]{ - // headFailureCount, operationType - { 1, OperationType.Create }, - { 2, OperationType.Create }, - { 3, OperationType.Create }, - { 4, OperationType.Create }, - { 1, OperationType.Read }, - { 2, OperationType.Read }, - { 4, OperationType.Read }, + // headFailureCount, operationType, successfulHeadRequestsWhichDontMeetBarrier + { 1, OperationType.Create, false, 0 }, + { 2, OperationType.Create, false, 0 }, + { 3, OperationType.Create, false, 0 }, + { 4, OperationType.Create, false, 0 }, + { 1, OperationType.Read, false, 0 }, + { 2, OperationType.Read, false, 0 }, + { 3, OperationType.Read, false, 0 }, + { 4, OperationType.Read, false, 0 }, + { 1, OperationType.Read, true, 18 }, + { 2, OperationType.Read, true, 18 }, + { 3, OperationType.Read, true, 18 }, + { 4, OperationType.Read, true, 18 }, + { 1, OperationType.Read, true, 108 }, + { 2, OperationType.Read, true, 108 }, + { 3, OperationType.Read, true, 108 }, + { 4, OperationType.Read, true, 108 } }; } - @Test(groups = {"multi-region"}, dataProvider = "headFailureScenarios" , timeOut = 2 * TIMEOUT) - public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, OperationType operationTypeForWhichBarrierFlowIsTriggered) throws Exception { + @Test(groups = {"multi-region"}, dataProvider = "headFailureScenarios" , timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) + public void validateGCLSNBarrierWithHeadFailures( + int headFailureCount, + OperationType operationTypeForWhichBarrierFlowIsTriggered, + boolean enterPostQuorumSelectionOnlyBarrierLoop, + int successfulHeadRequestCountWhichDontMeetBarrier) throws Exception { CosmosAsyncClient targetClient = getClientBuilder() .preferredRegions(this.preferredRegions) @@ -118,13 +133,15 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation ConnectionMode.DIRECT)) { safeClose(targetClient); + throw new SkipException("Skipping test for arguments: " + " OperationType: " + operationTypeForWhichBarrierFlowIsTriggered + " ConsistencyLevel: " + effectiveConsistencyLevel + " ConnectionMode: " + connectionModeOfClientUnderTest); } - AtomicInteger failureCountTracker = new AtomicInteger(); + AtomicInteger successfulHeadCountTracker = new AtomicInteger(); + AtomicInteger failedHeadCountTracker = new AtomicInteger(); Utils.ValueHolder originalRntbdTransportClientHolder = new Utils.ValueHolder<>(); RntbdTransportClientWithStoreResponseInterceptor interceptorClient = createClientWithInterceptor(targetClient, originalRntbdTransportClientHolder); @@ -134,28 +151,43 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation // Setup test data TestObject testObject = TestObject.create(); - if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Create) { - interceptorClient - .setResponseInterceptor( - StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( - operationTypeForWhichBarrierFlowIsTriggered, - this.regionNameToEndpoint.get(this.preferredRegions.get(1)), - headFailureCount, - failureCountTracker, - HttpConstants.StatusCodes.GONE, - HttpConstants.SubStatusCodes.LEASE_NOT_FOUND - )); - } else if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Read) { - interceptorClient - .setResponseInterceptor( - StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( - operationTypeForWhichBarrierFlowIsTriggered, - this.regionNameToEndpoint.get(this.preferredRegions.get(0)), - headFailureCount, - failureCountTracker, - HttpConstants.StatusCodes.GONE, - HttpConstants.SubStatusCodes.LEASE_NOT_FOUND - )); + if (enterPostQuorumSelectionOnlyBarrierLoop) { + if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Read) { + interceptorClient + .setResponseInterceptor( + StoreResponseInterceptorUtils.forceSuccessfulBarriersOnReadUntilQuorumSelectionThenForceBarrierFailures( + this.regionNameToEndpoint.get(this.preferredRegions.get(0)), + successfulHeadRequestCountWhichDontMeetBarrier, + successfulHeadCountTracker, + headFailureCount, + failedHeadCountTracker, + HttpConstants.StatusCodes.GONE, + HttpConstants.SubStatusCodes.LEASE_NOT_FOUND + )); + } + } else { + + if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Create) { + interceptorClient + .setResponseInterceptor( + StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( + this.regionNameToEndpoint.get(this.preferredRegions.get(1)), + headFailureCount, + failedHeadCountTracker, + HttpConstants.StatusCodes.GONE, + HttpConstants.SubStatusCodes.LEASE_NOT_FOUND + )); + } else if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Read) { + interceptorClient + .setResponseInterceptor( + StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( + this.regionNameToEndpoint.get(this.preferredRegions.get(0)), + headFailureCount, + failedHeadCountTracker, + HttpConstants.StatusCodes.GONE, + HttpConstants.SubStatusCodes.LEASE_NOT_FOUND + )); + } } try { @@ -181,7 +213,7 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation assertThat(diagnostics).isNotNull(); validateContactedRegions(diagnostics, 1); - validateHeadRequestsInCosmosDiagnostics(diagnostics, 2); + validateHeadRequestsInCosmosDiagnostics(diagnostics, 2, (2 + successfulHeadRequestCountWhichDontMeetBarrier)); } else { // Should timeout with 408 fail("Should have thrown timeout exception"); @@ -193,7 +225,7 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation CosmosItemResponse response = targetContainer.readItem(testObject.getId(), new PartitionKey(testObject.getMypk()), TestObject.class).block(); - if (headFailureCount <= 2) { + if (headFailureCount <= 3) { // Should eventually succeed assertThat(response).isNotNull(); assertThat(response.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.OK); @@ -203,7 +235,7 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation assertThat(diagnostics).isNotNull(); validateContactedRegions(diagnostics, 1); - validateHeadRequestsInCosmosDiagnostics(diagnostics, 4); + validateHeadRequestsInCosmosDiagnostics(diagnostics, 4, (4 + successfulHeadRequestCountWhichDontMeetBarrier)); } else { // Should eventually succeed assertThat(response).isNotNull(); @@ -214,7 +246,7 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation assertThat(diagnostics).isNotNull(); validateContactedRegions(diagnostics, 2); - validateHeadRequestsInCosmosDiagnostics(diagnostics, 4); + validateHeadRequestsInCosmosDiagnostics(diagnostics, 4, (4 + successfulHeadRequestCountWhichDontMeetBarrier)); } } @@ -231,7 +263,7 @@ public void validateGCLSNBarrierWithHeadFailures(int headFailureCount, Operation CosmosDiagnostics diagnostics = e.getDiagnostics(); validateContactedRegions(diagnostics, 1); - validateHeadRequestsInCosmosDiagnostics(diagnostics, 2); + validateHeadRequestsInCosmosDiagnostics(diagnostics, 2, (2 + successfulHeadRequestCountWhichDontMeetBarrier)); } } @@ -286,6 +318,23 @@ private RntbdTransportClientWithStoreResponseInterceptor createClientWithInterce private void validateContactedRegions(CosmosDiagnostics diagnostics, int expectedRegionsContactedCount) { CosmosDiagnosticsContext cosmosDiagnosticsContext = diagnostics.getDiagnosticsContext(); + Collection clientSideRequestStatisticsCollection + = diagnostics.getClientSideRequestStatistics(); + + for (ClientSideRequestStatistics clientSideRequestStatistics : clientSideRequestStatisticsCollection) { + + Collection storeResponseDiagnosticsList + = clientSideRequestStatistics.getResponseStatisticsList(); + + for (ClientSideRequestStatistics.StoreResponseStatistics storeResponseStatistics : storeResponseDiagnosticsList) { + if (storeResponseStatistics.getRequestOperationType() == OperationType.Create || storeResponseStatistics.getRequestOperationType() == OperationType.Read) { + + if (storeResponseStatistics.getStoreResult().getStoreResponseDiagnostics().getStatusCode() == 410) { + fail("Should not have encountered 410 for Create/Read operation"); + } + } + } + } assertThat(cosmosDiagnosticsContext).isNotNull(); assertThat(cosmosDiagnosticsContext.getContactedRegionNames()).isNotNull(); @@ -295,35 +344,65 @@ private void validateContactedRegions(CosmosDiagnostics diagnostics, int expecte private void validateHeadRequestsInCosmosDiagnostics( CosmosDiagnostics diagnostics, + int expectedHeadRequestCountWithFailures, int expectedHeadRequestCount) { - int headRequestCount = 0; + int actualHeadRequestCountWithLeaseNotFoundErrors = 0; + int actualHeadRequestCount = 0; boolean primaryReplicaContacted = false; Collection clientSideRequestStatisticsCollection = diagnostics.getClientSideRequestStatistics(); for (ClientSideRequestStatistics clientSideRequestStatistics : clientSideRequestStatisticsCollection) { + Collection storeResponseDiagnosticsList - = clientSideRequestStatistics.getSupplementalResponseStatisticsList(); + = clientSideRequestStatistics.getResponseStatisticsList(); for (ClientSideRequestStatistics.StoreResponseStatistics storeResponseStatistics : storeResponseDiagnosticsList) { + if (storeResponseStatistics.getRequestOperationType() == OperationType.Create || storeResponseStatistics.getRequestOperationType() == OperationType.Read) { + + if (storeResponseStatistics.getStoreResult().getStoreResponseDiagnostics().getStatusCode() == 410) { + fail("Should not have encountered 410 for Create/Read operation"); + } + } + } + + Collection supplementalResponseStatisticsList + = clientSideRequestStatistics.getSupplementalResponseStatisticsList(); + + for (ClientSideRequestStatistics.StoreResponseStatistics storeResponseStatistics : supplementalResponseStatisticsList) { if (storeResponseStatistics.getRequestOperationType() == OperationType.Head) { + + StoreResultDiagnostics storeResultDiagnostics = storeResponseStatistics.getStoreResult(); + + assertThat(storeResultDiagnostics).isNotNull(); + String storePhysicalAddressContacted - = storeResponseStatistics.getStoreResult().getStorePhysicalAddressAsString(); + = storeResultDiagnostics.getStorePhysicalAddressAsString(); + + StoreResponseDiagnostics storeResponseDiagnostics + = storeResultDiagnostics.getStoreResponseDiagnostics(); + + assertThat(storeResponseDiagnostics).isNotNull(); + + actualHeadRequestCount++; + + if (storeResponseDiagnostics.getStatusCode() == HttpConstants.StatusCodes.GONE && storeResponseDiagnostics.getSubStatusCode() == HttpConstants.SubStatusCodes.LEASE_NOT_FOUND) {; + actualHeadRequestCountWithLeaseNotFoundErrors++; + } if (isPrimaryReplicaEndpoint(storePhysicalAddressContacted)) { primaryReplicaContacted = true; } - - headRequestCount++; } } } assertThat(primaryReplicaContacted).isTrue(); - assertThat(headRequestCount).isGreaterThan(0); - assertThat(headRequestCount).isLessThanOrEqualTo(expectedHeadRequestCount); + assertThat(actualHeadRequestCountWithLeaseNotFoundErrors).isGreaterThan(0); + assertThat(actualHeadRequestCountWithLeaseNotFoundErrors).isLessThanOrEqualTo(expectedHeadRequestCountWithFailures); + assertThat(actualHeadRequestCount).isGreaterThanOrEqualTo(expectedHeadRequestCount); } private AccountLevelLocationContext getAccountLevelLocationContext(DatabaseAccount databaseAccount, boolean writeOnly) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java index 722d17d26900..18ff92f03894 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java @@ -15,7 +15,6 @@ public class StoreResponseInterceptorUtils { public static BiFunction forceBarrierFollowedByBarrierFailure( - OperationType operationType, String regionName, int maxAllowedFailureCount, AtomicInteger failureCount, @@ -30,6 +29,8 @@ public static BiFunction long manipulatedGCLSN = localLsn - 1; storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + + return storeResponse; } if (OperationType.Read.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { @@ -39,6 +40,8 @@ public static BiFunction long manipulatedGCLSN = Math.min(localLsn, itemLsn) - 1; storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + + return storeResponse; } if (OperationType.Head.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { @@ -50,4 +53,48 @@ public static BiFunction return storeResponse; }; } + + public static BiFunction forceSuccessfulBarriersOnReadUntilQuorumSelectionThenForceBarrierFailures(String regionName, + int allowedSuccessfulHeadRequestsWithoutBarrierBeingMet, + AtomicInteger successfulHeadRequestCount, + int maxAllowedFailureCount, + AtomicInteger failureCount, + int statusCode, + int subStatusCode) { + return (request, storeResponse) -> { + + if (OperationType.Read.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { + + long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); + long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); + long manipulatedGCLSN = Math.min(localLsn, itemLsn) - 1; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + + return storeResponse; + } + + if (OperationType.Head.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { + + if (successfulHeadRequestCount.incrementAndGet() <= allowedSuccessfulHeadRequestsWithoutBarrierBeingMet) { + + System.out.println("Allowing successful barrier for head request number: " + successfulHeadRequestCount.get()); + + long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); + long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); + long manipulatedGCLSN = Math.min(localLsn, itemLsn) - 1; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + + return storeResponse; + } + + if (failureCount.incrementAndGet() <= maxAllowedFailureCount) { + throw Utils.createCosmosException(statusCode, subStatusCode, new Exception("An intercepted exception occurred. Check status and substatus code for details."), null); + } + } + + return storeResponse; + }; + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java index b04eccb6c180..a56363369682 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java @@ -137,7 +137,6 @@ public Mono readStrongAsync( final MutableVolatile includePrimary = new MutableVolatile<>(false); final MutableVolatile cosmosExceptionValueHolder = new MutableVolatile<>(null); final MutableVolatile bailOnBarrierValueHolder = new MutableVolatile<>(false); - final AtomicInteger iterations = new AtomicInteger(0); return Flux.defer( // the following will be repeated till the repeat().takeUntil(.) condition is satisfied. @@ -192,8 +191,7 @@ public Mono readStrongAsync( secondaryQuorumReadResult.globalCommittedSelectedLsn, readMode, cosmosExceptionValueHolder, - bailOnBarrierValueHolder, - iterations); + bailOnBarrierValueHolder); return readBarrierObs.flux().flatMap( readBarrier -> { @@ -342,7 +340,7 @@ private Mono readQuorumAsync( Mono barrierRequestObs = BarrierRequestHelper.createAsync(this.diagnosticsClientContext, entity, this.authorizationTokenProvider, readLsn, globalCommittedLSN); return barrierRequestObs.flatMap( barrierRequest -> { - Mono waitForObs = this.waitForReadBarrierAsync(barrierRequest, false, readQuorum, readLsn, globalCommittedLSN, readMode, cosmosExceptionValueHolder, bailOnBarrierValueHolder, new AtomicInteger(0)); + Mono waitForObs = this.waitForReadBarrierAsync(barrierRequest, false, readQuorum, readLsn, globalCommittedLSN, readMode, cosmosExceptionValueHolder, bailOnBarrierValueHolder); return waitForObs.flatMap( waitFor -> { @@ -645,20 +643,14 @@ private Mono waitForReadBarrierAsync( final long targetGlobalCommittedLSN, ReadMode readMode, MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailFromReadBarrierLoopValueHolder, - AtomicInteger iterations) { + MutableVolatile bailFromReadBarrierLoopValueHolder) { AtomicInteger readBarrierRetryCount = new AtomicInteger(maxNumberOfReadBarrierReadRetries); AtomicInteger readBarrierRetryCountMultiRegion = new AtomicInteger(maxBarrierRetriesForMultiRegion); AtomicLong maxGlobalCommittedLsn = new AtomicLong(0); - AtomicLong repetitions = new AtomicLong(0); return Flux.defer(() -> { - iterations.incrementAndGet(); - - logger.warn("Iteration Count : {}", iterations.get()); - if (barrierRequest.requestContext.timeoutHelper.isElapsed()) { return Flux.error(new GoneException()); } @@ -745,12 +737,34 @@ private Mono waitForReadBarrierAsync( return Flux.error(new GoneException()); } - Mono> responsesObs = this.storeReader.readMultipleReplicaAsync( - barrierRequest, allowPrimary, readQuorum, - true /*required valid LSN*/, false /*useSessionToken*/, readMode, false /*checkMinLSN*/, true /*forceReadAll*/); + Mono> responsesObs = this.storeReader.readMultipleReplicaAsync( + barrierRequest, allowPrimary, readQuorum, + true /*required valid LSN*/, false /*useSessionToken*/, readMode, false /*checkMinLSN*/, true /*forceReadAll*/); return responsesObs.flux().flatMap( responses -> { + + boolean isAvoidQuorumSelectionStoreResult = false; + CosmosException cosmosExceptionFromStoreResult = null; + + for (StoreResult storeResult : responses) { + if (storeResult.isAvoidQuorumSelectionException) { + isAvoidQuorumSelectionStoreResult = true; + cosmosExceptionFromStoreResult = storeResult.getException(); + break; + } + } + + if (isAvoidQuorumSelectionStoreResult) { + return this.isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( + barrierRequest, + readBarrierLsn, + targetGlobalCommittedLSN, + cosmosExceptionValueHolder, + bailFromReadBarrierLoopValueHolder, + cosmosExceptionFromStoreResult); + } + long maxGlobalCommittedLsnInResponses = responses.size() > 0 ? responses.stream() .mapToLong(response -> response.globalCommittedLSN).max().getAsLong() : 0; From f0d333160e85817539667ac5c743bceeb57ee7f5 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 12 Nov 2025 17:13:19 -0500 Subject: [PATCH 05/30] Code comments --- .../cosmos/ExitFromConsistencyLayerTests.java | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index e772ce4e3e07..e48dae29f432 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -84,8 +84,24 @@ public ExitFromConsistencyLayerTests(CosmosClientBuilder clientBuilder) { super(clientBuilder); } - @DataProvider(name = "headFailureScenarios") - public static Object[][] headFailureScenarios() { +/** + * The data provider generates combinations of: + *
    + *
  • headFailureCount: Number of head failures to simulate
    • + *
  • operationTypeForWhichBarrierFlowIsTriggered: Operation type (Create/Read) for which barrier flow is triggered
    • + *
  • enterPostQuorumSelectionOnlyBarrierLoop: Whether to enter post quorum selection only barrier loop
    • + *
  • successfulHeadRequestsWhichDontMeetBarrier: Number of successful head requests which don't meet the barrier condition
    • + *
+ * + *

This helps cover scenarios where:

+ *
    + *
  • Enough head failures occur that the encapsulated document operation cannot succeed or has to be retried in a different region
    • + *
  • Head failures occur in the first phase of quorum selection
    • + *
  • Head failures occur in the second phase of quorum selection where the quorum was selected but not met
    • + *
+ */ + @DataProvider(name = "headRequestLeaseNotFoundScenarios") + public static Object[][] headRequestLeaseNotFoundScenarios() { return new Object[][]{ // headFailureCount, operationType, successfulHeadRequestsWhichDontMeetBarrier { 1, OperationType.Create, false, 0 }, @@ -107,8 +123,8 @@ public static Object[][] headFailureScenarios() { }; } - @Test(groups = {"multi-region"}, dataProvider = "headFailureScenarios" , timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) - public void validateGCLSNBarrierWithHeadFailures( + @Test(groups = {"multi-region"}, dataProvider = "headRequestLeaseNotFoundScenarios", timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) + public void validateHeadRequestLeaseNotFoundBailout( int headFailureCount, OperationType operationTypeForWhichBarrierFlowIsTriggered, boolean enterPostQuorumSelectionOnlyBarrierLoop, @@ -202,7 +218,7 @@ public void validateGCLSNBarrierWithHeadFailures( // Perform the operation CosmosItemResponse response = targetContainer.createItem(testObject).block(); - // For Create, Head can only fail up to 2 times + // For Create, Head can only fail up to 2 times before Create fails with a timeout if (headFailureCount <= 1) { // Should eventually succeed assertThat(response).isNotNull(); @@ -225,6 +241,7 @@ public void validateGCLSNBarrierWithHeadFailures( CosmosItemResponse response = targetContainer.readItem(testObject.getId(), new PartitionKey(testObject.getMypk()), TestObject.class).block(); + // for Read, Head can fail up to 3 times and still succeed from the same region after which read has to go to another region if (headFailureCount <= 3) { // Should eventually succeed assertThat(response).isNotNull(); @@ -241,7 +258,6 @@ public void validateGCLSNBarrierWithHeadFailures( assertThat(response).isNotNull(); assertThat(response.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.OK); - // Check diagnostics - should have contacted only one region for create CosmosDiagnostics diagnostics = response.getDiagnostics(); assertThat(diagnostics).isNotNull(); From e3b0db00b0f80bc7c81528c9b96332544196e032 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 14 Nov 2025 15:09:31 -0500 Subject: [PATCH 06/30] Adding a way to run tests against a multi-region Strong account. --- .../cosmos/ExitFromConsistencyLayerTests.java | 6 +++--- sdk/cosmos/live-platform-matrix.json | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index e48dae29f432..47603e5c2e19 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -53,7 +53,7 @@ public class ExitFromConsistencyLayerTests extends TestSuiteBase { private List preferredRegions; private Map regionNameToEndpoint; - @BeforeClass(groups = {"multi-region"}) + @BeforeClass(groups = {"fast"}) public void beforeClass() { try (CosmosAsyncClient dummy = getClientBuilder().buildAsyncClient()) { @@ -123,7 +123,7 @@ public static Object[][] headRequestLeaseNotFoundScenarios() { }; } - @Test(groups = {"multi-region"}, dataProvider = "headRequestLeaseNotFoundScenarios", timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) + @Test(groups = {"fast"}, dataProvider = "headRequestLeaseNotFoundScenarios", timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void validateHeadRequestLeaseNotFoundBailout( int headFailureCount, OperationType operationTypeForWhichBarrierFlowIsTriggered, @@ -479,6 +479,6 @@ public AccountLevelLocationContext( } } - @AfterClass(groups = {"multi-region"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + @AfterClass(groups = {"fast"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() {} } diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index 7f3e3af80997..c2b920f62c23 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -176,6 +176,20 @@ "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } + }, + { + "DESIRED_CONSISTENCIES": "[\"Strong\"]", + "ACCOUNT_CONSISTENCY": "Strong", + "ArmConfig": { + "MultiRegion_Strong": { + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Strong'; enableMultipleRegions = $true }" + } + }, + "PROTOCOLS": "[\"Tcp\"]", + "ProfileFlag": [ "-Pfast" ], + "Agent": { + "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } + } } ] } From 5d87a6f84da17577e1931752c1946f0b634ea08b Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 14 Nov 2025 17:32:15 -0500 Subject: [PATCH 07/30] Adding a way to run tests against a multi-region Strong account. --- sdk/cosmos/azure-cosmos-tests/pom.xml | 21 +++++++++++++++++++ .../cosmos/ExitFromConsistencyLayerTests.java | 6 +++--- .../com/azure/cosmos/rx/TestSuiteBase.java | 4 ++-- sdk/cosmos/live-platform-matrix.json | 3 ++- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index d02ae9502bdb..96e54e9d0057 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -789,5 +789,26 @@ Licensed under the MIT License. + + + multi-region-strong + + multi-region-strong + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.5.3 + + + src/test/resources/multi-region-strong.xml + + + + + + diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index 47603e5c2e19..99765e5e49ba 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -53,7 +53,7 @@ public class ExitFromConsistencyLayerTests extends TestSuiteBase { private List preferredRegions; private Map regionNameToEndpoint; - @BeforeClass(groups = {"fast"}) + @BeforeClass(groups = {"multi-region-strong"}) public void beforeClass() { try (CosmosAsyncClient dummy = getClientBuilder().buildAsyncClient()) { @@ -123,7 +123,7 @@ public static Object[][] headRequestLeaseNotFoundScenarios() { }; } - @Test(groups = {"fast"}, dataProvider = "headRequestLeaseNotFoundScenarios", timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) + @Test(groups = {"multi-region-strong"}, dataProvider = "headRequestLeaseNotFoundScenarios", timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void validateHeadRequestLeaseNotFoundBailout( int headFailureCount, OperationType operationTypeForWhichBarrierFlowIsTriggered, @@ -479,6 +479,6 @@ public AccountLevelLocationContext( } } - @AfterClass(groups = {"fast"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + @AfterClass(groups = {"multi-region-strong"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() {} } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index d17e872db25e..e10d4b33932c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -205,7 +205,7 @@ public CosmosAsyncDatabase getDatabase(String id) { @BeforeSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SETUP_TIMEOUT) + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong"}, timeOut = SUITE_SETUP_TIMEOUT) public void beforeSuite() { logger.info("beforeSuite Started"); @@ -230,7 +230,7 @@ public void parallelizeUnitTests(ITestContext context) { @AfterSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) public void afterSuite() { logger.info("afterSuite Started"); diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index c2b920f62c23..b291bcc0deab 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -13,6 +13,7 @@ "-Pcircuit-breaker-misc-gateway": "CircuitBreakerMiscGateway", "-Pcircuit-breaker-read-all-read-many": "CircuitBreakerReadAllAndReadMany", "-Pmulti-region": "MultiRegion", + "-Pmulti-region-strong": "MultiRegionStrong", "-Plong": "Long", "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"": "TCP", "Session": "", @@ -186,7 +187,7 @@ } }, "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pfast" ], + "ProfileFlag": [ "-Pmulti-region-strong" ], "Agent": { "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } From d46d9998b37b8d2b47d2250bfb1661c4a9315f5c Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 14 Nov 2025 17:54:20 -0500 Subject: [PATCH 08/30] Validate sub-status code too. --- .../java/com/azure/cosmos/ExitFromConsistencyLayerTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index 99765e5e49ba..473585aba517 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -273,8 +273,9 @@ public void validateHeadRequestLeaseNotFoundBailout( if (headFailureCount <= 1) { fail("Should have succeeded for create with head failures less than or equal to 2"); } else { - // Should get 408 timeout + // Should get 408-1022 timeout assertThat(e.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.REQUEST_TIMEOUT); + assertThat(e.getSubStatusCode()).isEqualTo(HttpConstants.SubStatusCodes.LEASE_NOT_FOUND); CosmosDiagnostics diagnostics = e.getDiagnostics(); From 77c9c5ad7ceb8f94649ea05e97796ae996a2ad70 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Fri, 14 Nov 2025 18:36:04 -0500 Subject: [PATCH 09/30] Code cleanup. --- .../cosmos/ExitFromConsistencyLayerTests.java | 32 ++++++++++++++ .../directconnectivity/StoreReader.java | 43 ++++++++----------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index 473585aba517..a8032f57b042 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -123,6 +123,38 @@ public static Object[][] headRequestLeaseNotFoundScenarios() { }; } + /** + * Validates that the consistency layer properly handles LEASE_NOT_FOUND (410-1022) failures during head requests + * in the barrier flow for strong consistency scenarios across multiple regions. + * + *

This test simulates various failure scenarios where head requests fail with LEASE_NOT_FOUND errors + * during the barrier protocol, which is used to ensure strong consistency guarantees. The test verifies + * that the client can properly bail out and retry when too many head failures occur, or successfully + * complete operations when head failures are within acceptable thresholds.

+ * + *

Test Scenarios:

+ *
    + *
  • For Create operations: Head can fail up to 1 time before the operation times out (408-1022)
    • + *
  • For Read operations: Head can fail up to 3 times and still succeed from the same region; + * with 4 failures, the operation fails over to another region
    • + *
  • Tests both pre-quorum and post-quorum selection barrier failure scenarios
    • + *
+ * + *

Verification Points:

+ *
    + *
  • Correct HTTP status codes (201 for Create, 200 for Read, 408 for timeouts)
    • + *
  • Proper region contact behavior (single region vs. failover to second region)
    • + *
  • Head request counts in diagnostics match expectations
    • + *
  • Primary replica is contacted during barrier protocol
    • + *
  • No 410 errors surface for Create/Read operations (should be handled internally)
    • + *
+ * + * @param headFailureCount Number of head requests that should fail with LEASE_NOT_FOUND + * @param operationTypeForWhichBarrierFlowIsTriggered The operation type (Create or Read) that triggers the barrier flow + * @param enterPostQuorumSelectionOnlyBarrierLoop Whether to simulate failures in the post-quorum selection phase + * @param successfulHeadRequestCountWhichDontMeetBarrier Number of successful head requests that don't meet the barrier condition + * @throws Exception if the test setup or execution fails + */ @Test(groups = {"multi-region-strong"}, dataProvider = "headRequestLeaseNotFoundScenarios", timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void validateHeadRequestLeaseNotFoundBailout( int headFailureCount, diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java index fd6fb4b6fc85..b3e17f406036 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java @@ -281,36 +281,31 @@ private Flux> readFromReplicas(List resultCollect if (srr.isAvoidQuorumSelectionException) { - // todo: fail fast when barrier requests also hit isAvoidQuorumSelectionException? - // todo: https://github.com/Azure/azure-sdk-for-java/issues/46135 - // if (!entity.isBarrierRequest) { + // isAvoidQuorumSelectionException is a special case where we want to enable the enclosing data plane operation + // to fail fast in the region where a quorum selection is being attempted + // no attempts to reselect quorum will be made + if (logger.isDebugEnabled()) { - // isAvoidQuorumSelectionException is a special case where we want to enable the enclosing data plane operation - // to fail fast in the region where a quorum selection is being attempted - // no attempts to reselect quorum will be made - if (logger.isDebugEnabled()) { + int statusCode = srr.getException() != null ? srr.getException().getStatusCode() : 0; + int subStatusCode = srr.getException() != null ? srr.getException().getSubStatusCode() : 0; - int statusCode = srr.getException() != null ? srr.getException().getStatusCode() : 0; - int subStatusCode = srr.getException() != null ? srr.getException().getSubStatusCode() : 0; - - logger.debug("An exception with error code [{}-{}] was observed which means quorum cannot be attained in the current region!", statusCode, subStatusCode); - } + logger.debug("An exception with error code [{}-{}] was observed which means quorum cannot be attained in the current region!", statusCode, subStatusCode); + } - if (!entity.requestContext.performedBackgroundAddressRefresh) { - this.startBackgroundAddressRefresh(entity); - entity.requestContext.performedBackgroundAddressRefresh = true; - } + if (!entity.requestContext.performedBackgroundAddressRefresh) { + this.startBackgroundAddressRefresh(entity); + entity.requestContext.performedBackgroundAddressRefresh = true; + } - // (collect quorum store results if possible) - // for QuorumReader (upstream) to make the final decision on quorum selection - resultCollector.add(srr); + // (collect quorum store results if possible) + // for QuorumReader (upstream) to make the final decision on quorum selection + resultCollector.add(srr); - // Remaining replicas - replicasToRead.set(replicaCountToRead - resultCollector.size()); + // Remaining replicas + replicasToRead.set(replicaCountToRead - resultCollector.size()); - // continue to the next store result - continue; - //} + // continue to the next store result + continue; } if (srr.isValid) { From 6e98e04fd58a1d327ffdb59a21f5ecbcbc7fdd81 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 17 Nov 2025 10:03:48 -0500 Subject: [PATCH 10/30] Add CHANGELOG.md entry. --- sdk/cosmos/azure-cosmos/CHANGELOG.md | 5 +++-- sdk/cosmos/live-platform-matrix.json | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/CHANGELOG.md b/sdk/cosmos/azure-cosmos/CHANGELOG.md index 5dedb4381709..4bda7cffc94c 100644 --- a/sdk/cosmos/azure-cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos/CHANGELOG.md @@ -7,10 +7,11 @@ #### Breaking Changes #### Bugs Fixed -* Fixed a possible memory leak (Netty buffers) in Gateway mode caused by a race condition when timeouts are happening. - [47228](https://github.com/Azure/azure-sdk-for-java/pull/47228) +* Fixed a possible memory leak (Netty buffers) in Gateway mode caused by a race condition when timeouts are happening. - [PR 47228](https://github.com/Azure/azure-sdk-for-java/pull/47228) #### Other Changes -* Changed to use incremental change feed to get partition key ranges. - [46810](https://github.com/Azure/azure-sdk-for-java/pull/46810) +* Changed to use incremental change feed to get partition key ranges. - [PR 46810](https://github.com/Azure/azure-sdk-for-java/pull/46810) +* Optimized 410 `Lease Not Found` handling for Strong Consistency account by avoiding unnecessary retries in the barrier attainment flow. - [PR 47232](https://github.com/Azure/azure-sdk-for-java/pull/47232) ### 4.75.0 (2025-10-21) > [!IMPORTANT] diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index b291bcc0deab..2d7abb1ce084 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -183,7 +183,7 @@ "ACCOUNT_CONSISTENCY": "Strong", "ArmConfig": { "MultiRegion_Strong": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Strong'; enableMultipleRegions = $true }" + "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong'; enableMultipleRegions = $true }" } }, "PROTOCOLS": "[\"Tcp\"]", From 2ef0be663999c7a48f7fc6c7ffe42a95e1e61e8a Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 17 Nov 2025 10:38:57 -0500 Subject: [PATCH 11/30] Modify barrier hit criteria. --- .../cosmos/ExitFromConsistencyLayerTests.java | 12 ++--- .../cosmos/StoreResponseInterceptorUtils.java | 45 ++++++++++++------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index a8032f57b042..4d08f4c9abc7 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -126,12 +126,12 @@ public static Object[][] headRequestLeaseNotFoundScenarios() { /** * Validates that the consistency layer properly handles LEASE_NOT_FOUND (410-1022) failures during head requests * in the barrier flow for strong consistency scenarios across multiple regions. - * + * *

This test simulates various failure scenarios where head requests fail with LEASE_NOT_FOUND errors * during the barrier protocol, which is used to ensure strong consistency guarantees. The test verifies * that the client can properly bail out and retry when too many head failures occur, or successfully * complete operations when head failures are within acceptable thresholds.

- * + * *

Test Scenarios:

*
    *
  • For Create operations: Head can fail up to 1 time before the operation times out (408-1022)
    • @@ -139,7 +139,7 @@ public static Object[][] headRequestLeaseNotFoundScenarios() { * with 4 failures, the operation fails over to another region *
  • Tests both pre-quorum and post-quorum selection barrier failure scenarios
    • *
- * + * *

Verification Points:

*
    *
  • Correct HTTP status codes (201 for Create, 200 for Read, 408 for timeouts)
    • @@ -148,7 +148,7 @@ public static Object[][] headRequestLeaseNotFoundScenarios() { *
  • Primary replica is contacted during barrier protocol
    • *
  • No 410 errors surface for Create/Read operations (should be handled internally)
    • *
- * + * * @param headFailureCount Number of head requests that should fail with LEASE_NOT_FOUND * @param operationTypeForWhichBarrierFlowIsTriggered The operation type (Create or Read) that triggers the barrier flow * @param enterPostQuorumSelectionOnlyBarrierLoop Whether to simulate failures in the post-quorum selection phase @@ -176,7 +176,7 @@ public void validateHeadRequestLeaseNotFoundBailout( if (!shouldTestExecutionHappen( effectiveConsistencyLevel, - ConsistencyLevel.STRONG, + OperationType.Create.equals(operationTypeForWhichBarrierFlowIsTriggered) ? ConsistencyLevel.STRONG : ConsistencyLevel.BOUNDED_STALENESS, connectionModeOfClientUnderTest, ConnectionMode.DIRECT)) { @@ -489,7 +489,7 @@ private boolean shouldTestExecutionHappen( return false; } - return accountConsistencyLevel.equals(minimumConsistencyLevel); + return accountConsistencyLevel.compareTo(minimumConsistencyLevel) <= 0; } private static boolean isPrimaryReplicaEndpoint(String storePhysicalAddress) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java index 18ff92f03894..2cfb95cd4364 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java @@ -25,21 +25,27 @@ public static BiFunction if (OperationType.Create.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { - long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); - long manipulatedGCLSN = localLsn - 1; + long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); + long manipulatedGlobalCommittedLSN = globalLsn - 1; - storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); return storeResponse; } if (OperationType.Read.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { - long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); + long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); - long manipulatedGCLSN = Math.min(localLsn, itemLsn) - 1; + long manipulatedGlobalCommittedLSN = Math.min(globalLsn, itemLsn) - 1; + long manipulatedGlobalLsn = itemLsn - 1; - storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + // Force barrier by setting GCLSN to be less than both local LSN and item LSN - applicable to strong consistency reads + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); + + // Force barrier by setting the (LSN - can correspond to GLSN when that particular document was part of a commit increment) + // to less than itemLsn - applicable to bounded staleness consistency reads + storeResponse.setHeaderValue(WFConstants.BackendHeaders.LSN, String.valueOf(manipulatedGlobalLsn)); return storeResponse; } @@ -64,12 +70,17 @@ public static BiFunction return (request, storeResponse) -> { if (OperationType.Read.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { - - long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); + long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); - long manipulatedGCLSN = Math.min(localLsn, itemLsn) - 1; + long manipulatedGlobalCommittedLSN = Math.min(globalLsn, itemLsn) - 1; + long manipulatedGlobalLsn = itemLsn - 1; + + // Force barrier by setting GCLSN to be less than both local LSN and item LSN - applicable to strong consistency reads + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); - storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + // Force barrier by setting the (LSN - can correspond to GLSN when that particular document was part of a commit increment) + // to less than itemLsn - applicable to bounded staleness consistency reads + storeResponse.setHeaderValue(WFConstants.BackendHeaders.LSN, String.valueOf(manipulatedGlobalLsn)); return storeResponse; } @@ -78,13 +89,17 @@ public static BiFunction if (successfulHeadRequestCount.incrementAndGet() <= allowedSuccessfulHeadRequestsWithoutBarrierBeingMet) { - System.out.println("Allowing successful barrier for head request number: " + successfulHeadRequestCount.get()); - - long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); + long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); - long manipulatedGCLSN = Math.min(localLsn, itemLsn) - 1; + long manipulatedGlobalCommittedLSN = Math.min(globalLsn, itemLsn) - 1; + long manipulatedGlobalLsn = itemLsn - 1; + + // Force barrier by setting GCLSN to be less than both local LSN and item LSN - applicable to strong consistency reads + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); - storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + // Force barrier by setting the (LSN - can correspond to GLSN when that particular document was part of a commit increment) + // to less than itemLsn - applicable to bounded staleness consistency reads + storeResponse.setHeaderValue(WFConstants.BackendHeaders.LSN, String.valueOf(manipulatedGlobalLsn)); return storeResponse; } From 42ef42999fc50927ed38f16c0c2269ecc18e2c84 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 17 Nov 2025 15:32:40 -0500 Subject: [PATCH 12/30] Modify barrier hit criteria. --- .../cosmos/ExitFromConsistencyLayerTests.java | 54 +++++---- .../cosmos/StoreResponseInterceptorUtils.java | 108 ++++++++++++------ .../test/resources/multi-region-strong.xml | 38 ++++++ 3 files changed, 141 insertions(+), 59 deletions(-) create mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-region-strong.xml diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index 4d08f4c9abc7..780b562deb95 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -15,6 +15,7 @@ import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.directconnectivity.ConsistencyReader; import com.azure.cosmos.implementation.directconnectivity.ConsistencyWriter; +//import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; import com.azure.cosmos.implementation.directconnectivity.ReplicatedResourceClient; import com.azure.cosmos.implementation.directconnectivity.RntbdTransportClient; @@ -24,6 +25,7 @@ import com.azure.cosmos.implementation.directconnectivity.StoreResultDiagnostics; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; +//import com.azure.cosmos.rx.TestSuiteBase; import com.azure.cosmos.rx.TestSuiteBase; import org.testng.SkipException; import org.testng.annotations.AfterClass; @@ -104,22 +106,23 @@ public ExitFromConsistencyLayerTests(CosmosClientBuilder clientBuilder) { public static Object[][] headRequestLeaseNotFoundScenarios() { return new Object[][]{ // headFailureCount, operationType, successfulHeadRequestsWhichDontMeetBarrier - { 1, OperationType.Create, false, 0 }, - { 2, OperationType.Create, false, 0 }, - { 3, OperationType.Create, false, 0 }, - { 4, OperationType.Create, false, 0 }, - { 1, OperationType.Read, false, 0 }, - { 2, OperationType.Read, false, 0 }, - { 3, OperationType.Read, false, 0 }, - { 4, OperationType.Read, false, 0 }, - { 1, OperationType.Read, true, 18 }, - { 2, OperationType.Read, true, 18 }, - { 3, OperationType.Read, true, 18 }, +// { 1, OperationType.Create, false, 0 }, +// { 2, OperationType.Create, false, 0 }, +// { 3, OperationType.Create, false, 0 }, +// { 4, OperationType.Create, false, 0 }, +// { 1, OperationType.Read, false, 0 }, +// { 2, OperationType.Read, false, 0 }, +// { 3, OperationType.Read, false, 0 }, +// { 4, OperationType.Read, false, 0 }, +// { 1, OperationType.Read, true, 18 }, +// { 2, OperationType.Read, true, 18 }, +// { 3, OperationType.Read, true, 18 }, { 4, OperationType.Read, true, 18 }, - { 1, OperationType.Read, true, 108 }, - { 2, OperationType.Read, true, 108 }, - { 3, OperationType.Read, true, 108 }, - { 4, OperationType.Read, true, 108 } + { 4, OperationType.Read, true, 18 } +// { 1, OperationType.Read, true, 108 }, +// { 2, OperationType.Read, true, 108 }, +// { 3, OperationType.Read, true, 108 }, +// { 4, OperationType.Read, true, 108 } }; } @@ -164,6 +167,8 @@ public void validateHeadRequestLeaseNotFoundBailout( CosmosAsyncClient targetClient = getClientBuilder() .preferredRegions(this.preferredRegions) + // revert + .consistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) .buildAsyncClient(); ConsistencyLevel effectiveConsistencyLevel @@ -204,6 +209,7 @@ public void validateHeadRequestLeaseNotFoundBailout( interceptorClient .setResponseInterceptor( StoreResponseInterceptorUtils.forceSuccessfulBarriersOnReadUntilQuorumSelectionThenForceBarrierFailures( + effectiveConsistencyLevel, this.regionNameToEndpoint.get(this.preferredRegions.get(0)), successfulHeadRequestCountWhichDontMeetBarrier, successfulHeadCountTracker, @@ -219,6 +225,7 @@ public void validateHeadRequestLeaseNotFoundBailout( interceptorClient .setResponseInterceptor( StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( + effectiveConsistencyLevel, this.regionNameToEndpoint.get(this.preferredRegions.get(1)), headFailureCount, failedHeadCountTracker, @@ -229,6 +236,7 @@ public void validateHeadRequestLeaseNotFoundBailout( interceptorClient .setResponseInterceptor( StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( + effectiveConsistencyLevel, this.regionNameToEndpoint.get(this.preferredRegions.get(0)), headFailureCount, failedHeadCountTracker, @@ -260,8 +268,8 @@ public void validateHeadRequestLeaseNotFoundBailout( CosmosDiagnostics diagnostics = response.getDiagnostics(); assertThat(diagnostics).isNotNull(); - validateContactedRegions(diagnostics, 1); validateHeadRequestsInCosmosDiagnostics(diagnostics, 2, (2 + successfulHeadRequestCountWhichDontMeetBarrier)); + validateContactedRegions(diagnostics, 1); } else { // Should timeout with 408 fail("Should have thrown timeout exception"); @@ -283,8 +291,8 @@ public void validateHeadRequestLeaseNotFoundBailout( CosmosDiagnostics diagnostics = response.getDiagnostics(); assertThat(diagnostics).isNotNull(); - validateContactedRegions(diagnostics, 1); validateHeadRequestsInCosmosDiagnostics(diagnostics, 4, (4 + successfulHeadRequestCountWhichDontMeetBarrier)); + validateContactedRegions(diagnostics, 1); } else { // Should eventually succeed assertThat(response).isNotNull(); @@ -293,8 +301,8 @@ public void validateHeadRequestLeaseNotFoundBailout( CosmosDiagnostics diagnostics = response.getDiagnostics(); assertThat(diagnostics).isNotNull(); - validateContactedRegions(diagnostics, 2); validateHeadRequestsInCosmosDiagnostics(diagnostics, 4, (4 + successfulHeadRequestCountWhichDontMeetBarrier)); + validateContactedRegions(diagnostics, 2); } } @@ -311,8 +319,8 @@ public void validateHeadRequestLeaseNotFoundBailout( CosmosDiagnostics diagnostics = e.getDiagnostics(); - validateContactedRegions(diagnostics, 1); validateHeadRequestsInCosmosDiagnostics(diagnostics, 2, (2 + successfulHeadRequestCountWhichDontMeetBarrier)); + validateContactedRegions(diagnostics, 1); } } @@ -448,10 +456,10 @@ private void validateHeadRequestsInCosmosDiagnostics( } } - assertThat(primaryReplicaContacted).isTrue(); - assertThat(actualHeadRequestCountWithLeaseNotFoundErrors).isGreaterThan(0); - assertThat(actualHeadRequestCountWithLeaseNotFoundErrors).isLessThanOrEqualTo(expectedHeadRequestCountWithFailures); - assertThat(actualHeadRequestCount).isGreaterThanOrEqualTo(expectedHeadRequestCount); + assertThat(actualHeadRequestCount).as("Not enough head requests were seen.").isGreaterThanOrEqualTo(expectedHeadRequestCount); + assertThat(actualHeadRequestCountWithLeaseNotFoundErrors).as("Head request failed count with 410/1022 should be greater than 1.").isGreaterThan(0); + assertThat(actualHeadRequestCountWithLeaseNotFoundErrors).as("Too many head request failed.").isLessThanOrEqualTo(expectedHeadRequestCountWithFailures); + assertThat(primaryReplicaContacted).as("Primary replica should be contacted even when a single Head request sees a 410/1022").isTrue(); } private AccountLevelLocationContext getAccountLevelLocationContext(DatabaseAccount databaseAccount, boolean writeOnly) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java index 2cfb95cd4364..78291ac2432e 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java @@ -15,6 +15,7 @@ public class StoreResponseInterceptorUtils { public static BiFunction forceBarrierFollowedByBarrierFailure( + ConsistencyLevel operationConsistencyLevel, String regionName, int maxAllowedFailureCount, AtomicInteger failureCount, @@ -25,27 +26,37 @@ public static BiFunction if (OperationType.Create.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { - long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); - long manipulatedGlobalCommittedLSN = globalLsn - 1; + long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); + long manipulatedGCLSN = localLsn - 1; - storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); return storeResponse; } if (OperationType.Read.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { - long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); - long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); - long manipulatedGlobalCommittedLSN = Math.min(globalLsn, itemLsn) - 1; - long manipulatedGlobalLsn = itemLsn - 1; + if (ConsistencyLevel.STRONG.equals(operationConsistencyLevel)) { - // Force barrier by setting GCLSN to be less than both local LSN and item LSN - applicable to strong consistency reads - storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); + long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); + long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); + long manipulatedGlobalCommittedLSN = Math.min(globalLsn, itemLsn) - 1; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); + + return storeResponse; + } else if (ConsistencyLevel.BOUNDED_STALENESS.equals(operationConsistencyLevel)) { + + long manipulatedItemLSN = -1; + long manipulatedGlobalLSN = 0; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.LSN, String.valueOf(manipulatedGlobalLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN, String.valueOf(manipulatedGlobalLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.ITEM_LSN, String.valueOf(manipulatedItemLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.ITEM_LOCAL_LSN, String.valueOf(manipulatedItemLSN)); - // Force barrier by setting the (LSN - can correspond to GLSN when that particular document was part of a commit increment) - // to less than itemLsn - applicable to bounded staleness consistency reads - storeResponse.setHeaderValue(WFConstants.BackendHeaders.LSN, String.valueOf(manipulatedGlobalLsn)); + return storeResponse; + } return storeResponse; } @@ -60,27 +71,40 @@ public static BiFunction }; } - public static BiFunction forceSuccessfulBarriersOnReadUntilQuorumSelectionThenForceBarrierFailures(String regionName, - int allowedSuccessfulHeadRequestsWithoutBarrierBeingMet, - AtomicInteger successfulHeadRequestCount, - int maxAllowedFailureCount, - AtomicInteger failureCount, - int statusCode, + public static BiFunction forceSuccessfulBarriersOnReadUntilQuorumSelectionThenForceBarrierFailures( + ConsistencyLevel operationConsistencyLevel, + String regionName, + int allowedSuccessfulHeadRequestsWithoutBarrierBeingMet, + AtomicInteger successfulHeadRequestCount, + int maxAllowedFailureCount, + AtomicInteger failureCount, + int statusCode, int subStatusCode) { return (request, storeResponse) -> { if (OperationType.Read.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { - long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); - long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); - long manipulatedGlobalCommittedLSN = Math.min(globalLsn, itemLsn) - 1; - long manipulatedGlobalLsn = itemLsn - 1; - // Force barrier by setting GCLSN to be less than both local LSN and item LSN - applicable to strong consistency reads - storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); + if (ConsistencyLevel.STRONG.equals(operationConsistencyLevel)) { - // Force barrier by setting the (LSN - can correspond to GLSN when that particular document was part of a commit increment) - // to less than itemLsn - applicable to bounded staleness consistency reads - storeResponse.setHeaderValue(WFConstants.BackendHeaders.LSN, String.valueOf(manipulatedGlobalLsn)); + long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); + long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); + long manipulatedGlobalCommittedLSN = Math.min(globalLsn, itemLsn) - 1; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); + + return storeResponse; + } else if (ConsistencyLevel.BOUNDED_STALENESS.equals(operationConsistencyLevel)) { + + long manipulatedItemLSN = -1; + long manipulatedGlobalLSN = 0; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.LSN, String.valueOf(manipulatedGlobalLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN, String.valueOf(manipulatedGlobalLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.ITEM_LSN, String.valueOf(manipulatedItemLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.ITEM_LOCAL_LSN, String.valueOf(manipulatedItemLSN)); + + return storeResponse; + } return storeResponse; } @@ -89,17 +113,29 @@ public static BiFunction if (successfulHeadRequestCount.incrementAndGet() <= allowedSuccessfulHeadRequestsWithoutBarrierBeingMet) { - long globalLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LSN)); - long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); - long manipulatedGlobalCommittedLSN = Math.min(globalLsn, itemLsn) - 1; - long manipulatedGlobalLsn = itemLsn - 1; + if (ConsistencyLevel.STRONG.equals(operationConsistencyLevel)) { + System.out.println("Allowing successful barrier for head request number: " + successfulHeadRequestCount.get()); - // Force barrier by setting GCLSN to be less than both local LSN and item LSN - applicable to strong consistency reads - storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGlobalCommittedLSN)); + long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); + long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); + long manipulatedGCLSN = Math.min(localLsn, itemLsn) - 1; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.GLOBAL_COMMITTED_LSN, String.valueOf(manipulatedGCLSN)); + + return storeResponse; + } else if (ConsistencyLevel.BOUNDED_STALENESS.equals(operationConsistencyLevel)) { + System.out.println("Allowing successful barrier for head request number: " + successfulHeadRequestCount.get()); + + long manipulatedItemLSN = -1; + long manipulatedGlobalLSN = -1; + + storeResponse.setHeaderValue(WFConstants.BackendHeaders.LSN, String.valueOf(manipulatedGlobalLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN, String.valueOf(manipulatedGlobalLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.ITEM_LSN, String.valueOf(manipulatedItemLSN)); + storeResponse.setHeaderValue(WFConstants.BackendHeaders.ITEM_LOCAL_LSN, String.valueOf(manipulatedItemLSN)); - // Force barrier by setting the (LSN - can correspond to GLSN when that particular document was part of a commit increment) - // to less than itemLsn - applicable to bounded staleness consistency reads - storeResponse.setHeaderValue(WFConstants.BackendHeaders.LSN, String.valueOf(manipulatedGlobalLsn)); + return storeResponse; + } return storeResponse; } diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-region-strong.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-region-strong.xml new file mode 100644 index 000000000000..9b8607cbf971 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-tests/src/test/resources/multi-region-strong.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + From ad13540e0bacc4558af6090c7b79a96b88a217d4 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 17 Nov 2025 20:17:09 -0500 Subject: [PATCH 13/30] Add tests for barrier bail out in Bounded Staleness consistency. --- .../cosmos/ExitFromConsistencyLayerTests.java | 227 +++++++++++------- .../cosmos/StoreResponseInterceptorUtils.java | 2 - 2 files changed, 146 insertions(+), 83 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index 780b562deb95..e4f16321ca30 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -15,7 +15,6 @@ import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.directconnectivity.ConsistencyReader; import com.azure.cosmos.implementation.directconnectivity.ConsistencyWriter; -//import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; import com.azure.cosmos.implementation.directconnectivity.ReplicatedResourceClient; import com.azure.cosmos.implementation.directconnectivity.RntbdTransportClient; @@ -25,7 +24,6 @@ import com.azure.cosmos.implementation.directconnectivity.StoreResultDiagnostics; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; -//import com.azure.cosmos.rx.TestSuiteBase; import com.azure.cosmos.rx.TestSuiteBase; import org.testng.SkipException; import org.testng.annotations.AfterClass; @@ -86,89 +84,152 @@ public ExitFromConsistencyLayerTests(CosmosClientBuilder clientBuilder) { super(clientBuilder); } -/** - * The data provider generates combinations of: - *
    - *
  • headFailureCount: Number of head failures to simulate
    • - *
  • operationTypeForWhichBarrierFlowIsTriggered: Operation type (Create/Read) for which barrier flow is triggered
    • - *
  • enterPostQuorumSelectionOnlyBarrierLoop: Whether to enter post quorum selection only barrier loop
    • - *
  • successfulHeadRequestsWhichDontMeetBarrier: Number of successful head requests which don't meet the barrier condition
    • - *
- * - *

This helps cover scenarios where:

- *
    - *
  • Enough head failures occur that the encapsulated document operation cannot succeed or has to be retried in a different region
    • - *
  • Head failures occur in the first phase of quorum selection
    • - *
  • Head failures occur in the second phase of quorum selection where the quorum was selected but not met
    • - *
- */ + + /** + * Provides test scenarios for head request lease not found (410/1022) error handling. + * + *

Each scenario is defined by the following parameters:

+ *
    + *
  • headFailureCount: Number of Head requests that should fail with 410/1022 status codes
    • + *
  • operationTypeForWhichBarrierFlowIsTriggered: The operation type (Create or Read) that triggers the barrier flow
    • + *
  • enterPostQuorumSelectionOnlyBarrierLoop: Whether to enter the post quorum selection only barrier loop
    • + *
  • successfulHeadRequestCountWhichDontMeetBarrier: Number of successful Head requests (204 status code) + * that don't meet barrier requirements before failures start
    • + *
  • isCrossRegionRetryExpected: Whether a cross-region retry is expected for the scenario
    • + *
  • desiredClientLevelConsistency: The desired client-level consistency (STRONG or BOUNDED_STALENESS)
    • + *
+ * + *

Important Notes:

+ *
    + *
  • Scenarios are scoped to cases where document requests succeeded but barrier flows were triggered + * (excludes QuorumNotMet scenarios which scope all barriers to primary)
    • + *
  • Create operations tolerate up to 2 Head failures before failing with timeout
    • + *
  • Read operations tolerate 3-5 Head failures before requiring cross-region retry
    • + *
  • In the QuorumSelected phase, successful Head requests that don't meet barrier requirements are tolerated: + *
      + *
    • Up to 18 successful Head requests for BOUNDED_STALENESS consistency
      • + *
    • Up to 111 successful Head requests for STRONG consistency
      • + *
    + * (See QuorumReader - maxNumberOfReadBarrierReadRetries and maxBarrierRetriesForMultiRegion)
    • + *
+ * + * @return array of test scenario parameters for head request lease not found error handling + */ @DataProvider(name = "headRequestLeaseNotFoundScenarios") public static Object[][] headRequestLeaseNotFoundScenarios() { + return new Object[][]{ - // headFailureCount, operationType, successfulHeadRequestsWhichDontMeetBarrier -// { 1, OperationType.Create, false, 0 }, -// { 2, OperationType.Create, false, 0 }, -// { 3, OperationType.Create, false, 0 }, -// { 4, OperationType.Create, false, 0 }, -// { 1, OperationType.Read, false, 0 }, -// { 2, OperationType.Read, false, 0 }, -// { 3, OperationType.Read, false, 0 }, -// { 4, OperationType.Read, false, 0 }, -// { 1, OperationType.Read, true, 18 }, -// { 2, OperationType.Read, true, 18 }, -// { 3, OperationType.Read, true, 18 }, - { 4, OperationType.Read, true, 18 }, - { 4, OperationType.Read, true, 18 } -// { 1, OperationType.Read, true, 108 }, -// { 2, OperationType.Read, true, 108 }, -// { 3, OperationType.Read, true, 108 }, -// { 4, OperationType.Read, true, 108 } + { 1, OperationType.Create, false, 0, false, null }, + { 2, OperationType.Create, false, 0, false, null }, + { 3, OperationType.Create, false, 0, false, null }, + { 4, OperationType.Create, false, 0, false, null }, + { 400, OperationType.Create, false, 0, false, null }, + { 1, OperationType.Read, false, 0, false, null }, + { 2, OperationType.Read, false, 0, false, null }, + { 3, OperationType.Read, false, 0, false, null }, + { 4, OperationType.Read, false, 0, true, null }, + { 400, OperationType.Read, false, 0, true, null }, + { 1, OperationType.Read, true, 18, false, ConsistencyLevel.BOUNDED_STALENESS }, + { 2, OperationType.Read, true, 18, false, ConsistencyLevel.BOUNDED_STALENESS }, + { 3, OperationType.Read, true, 18, false, ConsistencyLevel.BOUNDED_STALENESS }, + { 4, OperationType.Read, true, 18, false, ConsistencyLevel.BOUNDED_STALENESS }, + { 5, OperationType.Read, true, 18, true, ConsistencyLevel.BOUNDED_STALENESS }, + { 1, OperationType.Read, true, 18, false, ConsistencyLevel.STRONG }, + { 2, OperationType.Read, true, 18, false, ConsistencyLevel.STRONG }, + { 3, OperationType.Read, true, 18, false, ConsistencyLevel.STRONG }, + { 4, OperationType.Read, true, 18, true, ConsistencyLevel.STRONG }, + { 1, OperationType.Read, true, 111, false, ConsistencyLevel.STRONG }, + { 2, OperationType.Read, true, 111, false, ConsistencyLevel.STRONG }, + { 3, OperationType.Read, true, 111, false, ConsistencyLevel.STRONG }, + { 4, OperationType.Read, true, 111, false, ConsistencyLevel.STRONG }, + { 5, OperationType.Read, true, 111, true, ConsistencyLevel.STRONG } }; } /** - * Validates that the consistency layer properly handles LEASE_NOT_FOUND (410-1022) failures during head requests - * in the barrier flow for strong consistency scenarios across multiple regions. + * Validates that the consistency layer properly handles lease not found (410/1022) errors during + * barrier head requests and implements appropriate bailout/retry strategies based on the failure count, + * operation type, and consistency level. * - *

This test simulates various failure scenarios where head requests fail with LEASE_NOT_FOUND errors - * during the barrier protocol, which is used to ensure strong consistency guarantees. The test verifies - * that the client can properly bail out and retry when too many head failures occur, or successfully - * complete operations when head failures are within acceptable thresholds.

+ *

This test verifies the resilience and fault tolerance of the barrier flow mechanism in the + * consistency layer when head requests fail with lease not found errors. The barrier flow is triggered + * when documents requests succeed but additional head requests are needed to ensure consistency guarantees + * are met across replicas.

* *

Test Scenarios:

*
    - *
  • For Create operations: Head can fail up to 1 time before the operation times out (408-1022)
    • - *
  • For Read operations: Head can fail up to 3 times and still succeed from the same region; - * with 4 failures, the operation fails over to another region
    • - *
  • Tests both pre-quorum and post-quorum selection barrier failure scenarios
    • + *
  • Create Operations: Can tolerate up to 2 head failures before timing out. Beyond this threshold, + * the operation fails with a 408 timeout status code with 1022 substatus.
    • + *
  • Read Operations: Can tolerate 3-4 head failures within the same region. After 4 failures, + * the operation triggers a cross-region retry to ensure eventual consistency.
    • + *
  • Post-Quorum Selection Barrier Loop: Tests scenarios where initial barriers pass but subsequent + * barriers in the quorum-selected phase encounter failures. The system tolerates different numbers of + * successful head requests that don't meet barrier requirements: + *
      + *
    • Up to 18 for BOUNDED_STALENESS consistency
      • + *
    • Up to 111 for STRONG consistency
      • + *
    + *
    • *
* - *

Verification Points:

+ *

Validation Steps:

+ *
    + *
  1. Creates a CosmosAsyncClient with the specified consistency level and preferred regions
    2. + *
  2. Intercepts store responses to inject controlled head request failures (410/1022)
    3. + *
  3. Executes the specified operation (Create or Read) that triggers the barrier flow
    4. + *
  4. Validates the operation completes successfully or fails appropriately based on failure count
    5. + *
  5. Examines diagnostics to verify: + *
      + *
    • Correct number of head requests were attempted
      • + *
    • Expected number of regions were contacted
      • + *
    • Primary replica was contacted when failures occurred
      • + *
    • No 410 errors reached the Create/Read operations themselves
      • + *
    + *
    6. + *
+ * + *

Important Notes:

*
    - *
  • Correct HTTP status codes (201 for Create, 200 for Read, 408 for timeouts)
    • - *
  • Proper region contact behavior (single region vs. failover to second region)
    • - *
  • Head request counts in diagnostics match expectations
    • - *
  • Primary replica is contacted during barrier protocol
    • - *
  • No 410 errors surface for Create/Read operations (should be handled internally)
    • + *
  • Test scenarios are scoped to cases where document requests succeeded but barrier flows were triggered, + * excluding QuorumNotMet scenarios which scope all barriers to the primary region
    • + *
  • The test uses an interceptor client to inject failures at precise points in the barrier flow
    • + *
  • Cross-region retry is only expected for Read operations exceeding the failure threshold
    • + *
  • The test validates that the system never exposes 410/1022 errors to the application layer for + * the primary Create/Read operations - only head requests should encounter these errors
    • *
* - * @param headFailureCount Number of head requests that should fail with LEASE_NOT_FOUND - * @param operationTypeForWhichBarrierFlowIsTriggered The operation type (Create or Read) that triggers the barrier flow - * @param enterPostQuorumSelectionOnlyBarrierLoop Whether to simulate failures in the post-quorum selection phase - * @param successfulHeadRequestCountWhichDontMeetBarrier Number of successful head requests that don't meet the barrier condition - * @throws Exception if the test setup or execution fails + * @param headFailureCount the number of head requests that should fail with 410/1022 status codes + * before the system either succeeds, times out, or retries in another region + * @param operationTypeForWhichBarrierFlowIsTriggered the type of operation (Create or Read) that triggers + * the barrier flow and determines retry behavior + * @param enterPostQuorumSelectionOnlyBarrierLoop if true, allows initial barriers to pass and only injects + * failures during the post-quorum selection barrier loop phase + * @param successfulHeadRequestCountWhichDontMeetBarrier the number of successful head requests (204 status) + * that don't meet barrier requirements before failures + * start being injected (only relevant when in post-quorum + * selection barrier loop) + * @param isCrossRegionRetryExpected whether the test expects a cross-region retry to occur based on the + * failure count and operation type + * @param consistencyLevelApplicableForTest the consistency level to configure on the test client (STRONG or + * BOUNDED_STALENESS), which affects barrier retry thresholds + * + * @throws Exception if the test setup fails or unexpected errors occur during test execution + * + * @see ConsistencyReader for barrier flow implementation in read path + * @see ConsistencyWriter for barrier flow implementation in write path + * @see StoreResponseInterceptorUtils for failure injection mechanisms */ @Test(groups = {"multi-region-strong"}, dataProvider = "headRequestLeaseNotFoundScenarios", timeOut = 2 * TIMEOUT, retryAnalyzer = FlakyTestRetryAnalyzer.class) public void validateHeadRequestLeaseNotFoundBailout( int headFailureCount, OperationType operationTypeForWhichBarrierFlowIsTriggered, boolean enterPostQuorumSelectionOnlyBarrierLoop, - int successfulHeadRequestCountWhichDontMeetBarrier) throws Exception { + int successfulHeadRequestCountWhichDontMeetBarrier, + boolean isCrossRegionRetryExpected, + ConsistencyLevel consistencyLevelApplicableForTest) throws Exception { CosmosAsyncClient targetClient = getClientBuilder() .preferredRegions(this.preferredRegions) - // revert - .consistencyLevel(ConsistencyLevel.BOUNDED_STALENESS) .buildAsyncClient(); ConsistencyLevel effectiveConsistencyLevel @@ -182,15 +243,16 @@ public void validateHeadRequestLeaseNotFoundBailout( if (!shouldTestExecutionHappen( effectiveConsistencyLevel, OperationType.Create.equals(operationTypeForWhichBarrierFlowIsTriggered) ? ConsistencyLevel.STRONG : ConsistencyLevel.BOUNDED_STALENESS, - connectionModeOfClientUnderTest, - ConnectionMode.DIRECT)) { + consistencyLevelApplicableForTest, + connectionModeOfClientUnderTest)) { safeClose(targetClient); throw new SkipException("Skipping test for arguments: " + " OperationType: " + operationTypeForWhichBarrierFlowIsTriggered + " ConsistencyLevel: " + effectiveConsistencyLevel + - " ConnectionMode: " + connectionModeOfClientUnderTest); + " ConnectionMode: " + connectionModeOfClientUnderTest + + " DesiredConsistencyLevel: " + consistencyLevelApplicableForTest); } AtomicInteger successfulHeadCountTracker = new AtomicInteger(); @@ -201,11 +263,10 @@ public void validateHeadRequestLeaseNotFoundBailout( try { - // Setup test data TestObject testObject = TestObject.create(); if (enterPostQuorumSelectionOnlyBarrierLoop) { - if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Read) { + if (OperationType.Read.equals(operationTypeForWhichBarrierFlowIsTriggered)) { interceptorClient .setResponseInterceptor( StoreResponseInterceptorUtils.forceSuccessfulBarriersOnReadUntilQuorumSelectionThenForceBarrierFailures( @@ -221,7 +282,7 @@ public void validateHeadRequestLeaseNotFoundBailout( } } else { - if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Create) { + if (OperationType.Create.equals(operationTypeForWhichBarrierFlowIsTriggered)) { interceptorClient .setResponseInterceptor( StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( @@ -232,7 +293,7 @@ public void validateHeadRequestLeaseNotFoundBailout( HttpConstants.StatusCodes.GONE, HttpConstants.SubStatusCodes.LEASE_NOT_FOUND )); - } else if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Read) { + } else if (OperationType.Read.equals(operationTypeForWhichBarrierFlowIsTriggered)) { interceptorClient .setResponseInterceptor( StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( @@ -282,7 +343,7 @@ public void validateHeadRequestLeaseNotFoundBailout( = targetContainer.readItem(testObject.getId(), new PartitionKey(testObject.getMypk()), TestObject.class).block(); // for Read, Head can fail up to 3 times and still succeed from the same region after which read has to go to another region - if (headFailureCount <= 3) { + if (!isCrossRegionRetryExpected) { // Should eventually succeed assertThat(response).isNotNull(); assertThat(response.getStatusCode()).isEqualTo(HttpConstants.StatusCodes.OK); @@ -291,7 +352,7 @@ public void validateHeadRequestLeaseNotFoundBailout( CosmosDiagnostics diagnostics = response.getDiagnostics(); assertThat(diagnostics).isNotNull(); - validateHeadRequestsInCosmosDiagnostics(diagnostics, 4, (4 + successfulHeadRequestCountWhichDontMeetBarrier)); + validateHeadRequestsInCosmosDiagnostics(diagnostics, 5, (5 + successfulHeadRequestCountWhichDontMeetBarrier)); validateContactedRegions(diagnostics, 1); } else { // Should eventually succeed @@ -301,7 +362,7 @@ public void validateHeadRequestLeaseNotFoundBailout( CosmosDiagnostics diagnostics = response.getDiagnostics(); assertThat(diagnostics).isNotNull(); - validateHeadRequestsInCosmosDiagnostics(diagnostics, 4, (4 + successfulHeadRequestCountWhichDontMeetBarrier)); + validateHeadRequestsInCosmosDiagnostics(diagnostics, 5, (5 + successfulHeadRequestCountWhichDontMeetBarrier)); validateContactedRegions(diagnostics, 2); } } @@ -325,7 +386,7 @@ public void validateHeadRequestLeaseNotFoundBailout( } if (operationTypeForWhichBarrierFlowIsTriggered == OperationType.Read) { - fail("Read operation should have succeeded even with head failures"); + fail("Read operation should have succeeded even with head failures through cross region retry."); } } @@ -396,13 +457,13 @@ private void validateContactedRegions(CosmosDiagnostics diagnostics, int expecte assertThat(cosmosDiagnosticsContext).isNotNull(); assertThat(cosmosDiagnosticsContext.getContactedRegionNames()).isNotNull(); assertThat(cosmosDiagnosticsContext.getContactedRegionNames()).isNotEmpty(); - assertThat(cosmosDiagnosticsContext.getContactedRegionNames().size()).isEqualTo(expectedRegionsContactedCount); + assertThat(cosmosDiagnosticsContext.getContactedRegionNames().size()).as("Mismatch in regions contacted.").isEqualTo(expectedRegionsContactedCount); } private void validateHeadRequestsInCosmosDiagnostics( CosmosDiagnostics diagnostics, - int expectedHeadRequestCountWithFailures, - int expectedHeadRequestCount) { + int maxExpectedHeadRequestCountWithLeaseNotFoundErrors, + int maxExpectedHeadRequestCount) { int actualHeadRequestCountWithLeaseNotFoundErrors = 0; int actualHeadRequestCount = 0; @@ -456,9 +517,9 @@ private void validateHeadRequestsInCosmosDiagnostics( } } - assertThat(actualHeadRequestCount).as("Not enough head requests were seen.").isGreaterThanOrEqualTo(expectedHeadRequestCount); assertThat(actualHeadRequestCountWithLeaseNotFoundErrors).as("Head request failed count with 410/1022 should be greater than 1.").isGreaterThan(0); - assertThat(actualHeadRequestCountWithLeaseNotFoundErrors).as("Too many head request failed.").isLessThanOrEqualTo(expectedHeadRequestCountWithFailures); + assertThat(actualHeadRequestCountWithLeaseNotFoundErrors).as("Too many head request failed.").isLessThanOrEqualTo(maxExpectedHeadRequestCountWithLeaseNotFoundErrors); + assertThat(actualHeadRequestCount).as("Too many Head requests made perhaps due to real replication lag.").isLessThanOrEqualTo(maxExpectedHeadRequestCount + 10); assertThat(primaryReplicaContacted).as("Primary replica should be contacted even when a single Head request sees a 410/1022").isTrue(); } @@ -488,16 +549,20 @@ private AccountLevelLocationContext getAccountLevelLocationContext(DatabaseAccou } private boolean shouldTestExecutionHappen( - ConsistencyLevel accountConsistencyLevel, + ConsistencyLevel effectiveConsistencyLevel, ConsistencyLevel minimumConsistencyLevel, - ConnectionMode connectionModeOfClientUnderTest, - ConnectionMode expectedConnectionMode) { + ConsistencyLevel consistencyLevelApplicableForTestScenario, + ConnectionMode connectionModeOfClientUnderTest) { - if (!connectionModeOfClientUnderTest.name().equalsIgnoreCase(expectedConnectionMode.name())) { + if (!connectionModeOfClientUnderTest.name().equalsIgnoreCase(ConnectionMode.DIRECT.name())) { return false; } - return accountConsistencyLevel.compareTo(minimumConsistencyLevel) <= 0; + if (consistencyLevelApplicableForTestScenario != null) { + return consistencyLevelApplicableForTestScenario.equals(effectiveConsistencyLevel); + } + + return effectiveConsistencyLevel.compareTo(minimumConsistencyLevel) <= 0; } private static boolean isPrimaryReplicaEndpoint(String storePhysicalAddress) { diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java index 78291ac2432e..d8387f6e63db 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java @@ -114,7 +114,6 @@ public static BiFunction if (successfulHeadRequestCount.incrementAndGet() <= allowedSuccessfulHeadRequestsWithoutBarrierBeingMet) { if (ConsistencyLevel.STRONG.equals(operationConsistencyLevel)) { - System.out.println("Allowing successful barrier for head request number: " + successfulHeadRequestCount.get()); long localLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.LOCAL_LSN)); long itemLsn = Long.parseLong(storeResponse.getHeaderValue(WFConstants.BackendHeaders.ITEM_LSN)); @@ -124,7 +123,6 @@ public static BiFunction return storeResponse; } else if (ConsistencyLevel.BOUNDED_STALENESS.equals(operationConsistencyLevel)) { - System.out.println("Allowing successful barrier for head request number: " + successfulHeadRequestCount.get()); long manipulatedItemLSN = -1; long manipulatedGlobalLSN = -1; From 41cec42a6daffe8c6f8e30d86a0b00ec3aadcb13 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 17 Nov 2025 20:20:14 -0500 Subject: [PATCH 14/30] Add tests for barrier bail out in Bounded Staleness consistency. --- .../azure/cosmos/ExitFromConsistencyLayerTests.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index e4f16321ca30..7eb90e233868 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -12,7 +12,9 @@ import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.RxDocumentClientImpl; +import com.azure.cosmos.implementation.Strings; import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.apachecommons.codec.binary.StringUtils; import com.azure.cosmos.implementation.directconnectivity.ConsistencyReader; import com.azure.cosmos.implementation.directconnectivity.ConsistencyWriter; import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; @@ -510,7 +512,7 @@ private void validateHeadRequestsInCosmosDiagnostics( actualHeadRequestCountWithLeaseNotFoundErrors++; } - if (isPrimaryReplicaEndpoint(storePhysicalAddressContacted)) { + if (isStorePhysicalAddressThatOfPrimaryReplica(storePhysicalAddressContacted)) { primaryReplicaContacted = true; } } @@ -565,7 +567,12 @@ private boolean shouldTestExecutionHappen( return effectiveConsistencyLevel.compareTo(minimumConsistencyLevel) <= 0; } - private static boolean isPrimaryReplicaEndpoint(String storePhysicalAddress) { + private static boolean isStorePhysicalAddressThatOfPrimaryReplica(String storePhysicalAddress) { + + if (Strings.isNullOrEmpty(storePhysicalAddress)) { + return false; + } + return storePhysicalAddress.endsWith("p/"); } From e4f93b02ca361212bf2cc6765023ee788e93b124 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 17 Nov 2025 20:29:29 -0500 Subject: [PATCH 15/30] Refactoring --- .../java/com/azure/cosmos/ExitFromConsistencyLayerTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index 7eb90e233868..b1305c3ef0ae 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -14,7 +14,6 @@ import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.Strings; import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.implementation.apachecommons.codec.binary.StringUtils; import com.azure.cosmos.implementation.directconnectivity.ConsistencyReader; import com.azure.cosmos.implementation.directconnectivity.ConsistencyWriter; import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; From 23a8dee7adfa790cdc70ffad40e7e31f1f3e4c56 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 18 Nov 2025 09:34:46 -0500 Subject: [PATCH 16/30] Addressing code comments. --- .../java/com/azure/cosmos/ExitFromConsistencyLayerTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java index b1305c3ef0ae..ab29561b38f4 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java @@ -507,7 +507,7 @@ private void validateHeadRequestsInCosmosDiagnostics( actualHeadRequestCount++; - if (storeResponseDiagnostics.getStatusCode() == HttpConstants.StatusCodes.GONE && storeResponseDiagnostics.getSubStatusCode() == HttpConstants.SubStatusCodes.LEASE_NOT_FOUND) {; + if (storeResponseDiagnostics.getStatusCode() == HttpConstants.StatusCodes.GONE && storeResponseDiagnostics.getSubStatusCode() == HttpConstants.SubStatusCodes.LEASE_NOT_FOUND) { actualHeadRequestCountWithLeaseNotFoundErrors++; } From cff675e363bde9a38d1f640060d261f490828cdb Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Tue, 18 Nov 2025 09:39:49 -0500 Subject: [PATCH 17/30] Verify write barrier criteria. --- ...istencyLayerTests.java => BailOutFromBarrierE2ETests.java} | 4 ++-- .../implementation/directconnectivity/ConsistencyWriter.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) rename sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/{ExitFromConsistencyLayerTests.java => BailOutFromBarrierE2ETests.java} (99%) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java similarity index 99% rename from sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java rename to sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java index ab29561b38f4..33c04b84bdf6 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/ExitFromConsistencyLayerTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java @@ -44,7 +44,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; -public class ExitFromConsistencyLayerTests extends TestSuiteBase { +public class BailOutFromBarrierE2ETests extends TestSuiteBase { private static final ImplementationBridgeHelpers.CosmosAsyncClientHelper.CosmosAsyncClientAccessor cosmosAsyncClientAccessor = ImplementationBridgeHelpers.CosmosAsyncClientHelper.getCosmosAsyncClientAccessor(); @@ -81,7 +81,7 @@ public void beforeClass() { } @Factory(dataProvider = "clientBuildersWithDirectTcp") - public ExitFromConsistencyLayerTests(CosmosClientBuilder clientBuilder) { + public BailOutFromBarrierE2ETests(CosmosClientBuilder clientBuilder) { super(clientBuilder); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java index c4f5ee117caf..3dd0a5f562ef 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java @@ -545,8 +545,7 @@ private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied return Mono.just(false); } - boolean hasRequiredGlobalCommittedLsn = - selectedGlobalCommittedLSN <= 0 || storeResult.globalCommittedLSN >= selectedGlobalCommittedLSN; + boolean hasRequiredGlobalCommittedLsn = storeResult.globalCommittedLSN >= selectedGlobalCommittedLSN; return Mono.just(hasRequiredGlobalCommittedLsn); }) From 0fc232bbff78864c0af421e3cacb123a0942e876 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 19 Nov 2025 12:38:43 -0500 Subject: [PATCH 18/30] Verify barrier bail out criteria. --- .../directconnectivity/ConsistencyWriter.java | 77 +++++++++++++------ .../directconnectivity/QuorumReader.java | 54 ++++++++++--- 2 files changed, 96 insertions(+), 35 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java index 3dd0a5f562ef..d820380b217f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java @@ -283,7 +283,7 @@ Mono writePrivateAsync( false, primaryURI.get(), replicaStatusList); - return barrierForGlobalStrong(request, response, cosmosExceptionValueHolder, bailFromWriteBarrierLoopValueHolder); + return barrierForGlobalStrong(request, response, cosmosExceptionValueHolder); }) .doFinally(signalType -> { if (signalType != SignalType.CANCEL) { @@ -299,7 +299,7 @@ Mono writePrivateAsync( } else { Mono barrierRequestObs = BarrierRequestHelper.createAsync(this.diagnosticsClientContext, request, this.authorizationTokenProvider, null, request.requestContext.globalCommittedSelectedLSN); - return barrierRequestObs.flatMap(barrierRequest -> waitForWriteBarrierAsync(barrierRequest, request.requestContext.globalCommittedSelectedLSN, cosmosExceptionValueHolder, bailFromWriteBarrierLoopValueHolder) + return barrierRequestObs.flatMap(barrierRequest -> waitForWriteBarrierAsync(barrierRequest, request.requestContext.globalCommittedSelectedLSN, cosmosExceptionValueHolder) .flatMap(v -> { if (!v) { @@ -335,8 +335,7 @@ boolean isGlobalStrongRequest(RxDocumentServiceRequest request, StoreResponse re Mono barrierForGlobalStrong( RxDocumentServiceRequest request, StoreResponse response, - MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailFromWriteBarrierLoopValueHolder) { + MutableVolatile cosmosExceptionValueHolder) { try { if (ReplicatedResourceClient.isGlobalStrongEnabled() && this.isGlobalStrongRequest(request, response)) { @@ -368,7 +367,7 @@ Mono barrierForGlobalStrong( request.requestContext.globalCommittedSelectedLSN); return barrierRequestObs.flatMap(barrierRequest -> { - Mono barrierWait = this.waitForWriteBarrierAsync(barrierRequest, request.requestContext.globalCommittedSelectedLSN, cosmosExceptionValueHolder, bailFromWriteBarrierLoopValueHolder); + Mono barrierWait = this.waitForWriteBarrierAsync(barrierRequest, request.requestContext.globalCommittedSelectedLSN, cosmosExceptionValueHolder); return barrierWait.flatMap(res -> { if (!res) { @@ -405,8 +404,7 @@ Mono barrierForGlobalStrong( private Mono waitForWriteBarrierAsync( RxDocumentServiceRequest barrierRequest, long selectedGlobalCommittedLsn, - MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailFromWriteBarrierLoop) { + MutableVolatile cosmosExceptionValueHolder) { AtomicInteger writeBarrierRetryCount = new AtomicInteger(ConsistencyWriter.MAX_NUMBER_OF_WRITE_BARRIER_READ_RETRIES); AtomicLong maxGlobalCommittedLsnReceived = new AtomicLong(0); @@ -439,11 +437,11 @@ private Mono waitForWriteBarrierAsync( } if (isAvoidQuorumSelectionStoreResult) { + writeBarrierRetryCount.getAndIncrement(); return this.isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( barrierRequest, selectedGlobalCommittedLsn, cosmosExceptionValueHolder, - bailFromWriteBarrierLoop, cosmosExceptionFromStoreResult); } @@ -507,21 +505,24 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep RxDocumentServiceRequest barrierRequest, long selectedGlobalCommittedLsn, MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailFromWriteBarrierLoop, CosmosException cosmosExceptionInStoreResult) { - return performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( + MutableVolatile bailFromWriteBarrierLoop = new MutableVolatile<>(false); + + return performOptimisticBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( barrierRequest, - selectedGlobalCommittedLsn) - .flatMap(barrierStatusFromPrimary -> { + selectedGlobalCommittedLsn, + cosmosExceptionValueHolder, + bailFromWriteBarrierLoop).flatMap(isBarrierFromPrimarySuccessful -> { - if (barrierStatusFromPrimary) { - bailFromWriteBarrierLoop.v = true; - cosmosExceptionValueHolder.v = null; + if (isBarrierFromPrimarySuccessful) { + bailFromWriteBarrierLoop.v = true; + cosmosExceptionValueHolder.v = null; - return Mono.just(true); - } + return Mono.just(true); + } + if (bailFromWriteBarrierLoop.v) { bailFromWriteBarrierLoop.v = true; cosmosExceptionValueHolder.v = Utils.createCosmosException( HttpConstants.StatusCodes.REQUEST_TIMEOUT, @@ -529,16 +530,23 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep cosmosExceptionInStoreResult, null); return Mono.just(false); - }); + } else { + bailFromWriteBarrierLoop.v = false; + cosmosExceptionValueHolder.v = null; + return Mono.empty(); + } + }); } - private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( - RxDocumentServiceRequest entity, - long selectedGlobalCommittedLSN) { + private Mono performOptimisticBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( + RxDocumentServiceRequest barrierRequest, + long selectedGlobalCommittedLSN, + MutableVolatile cosmosExceptionValueHolder, + MutableVolatile bailFromWriteBarrierLoop) { - entity.requestContext.forceRefreshAddressCache = true; + barrierRequest.requestContext.forceRefreshAddressCache = true; Mono storeResultObs = this.storeReader.readPrimaryAsync( - entity, true, false /*useSessionToken*/); + barrierRequest, true, false /*useSessionToken*/); return storeResultObs.flatMap(storeResult -> { if (!storeResult.isValid) { @@ -547,9 +555,30 @@ private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied boolean hasRequiredGlobalCommittedLsn = storeResult.globalCommittedLSN >= selectedGlobalCommittedLSN; + barrierRequest.requestContext.forceRefreshAddressCache = false; return Mono.just(hasRequiredGlobalCommittedLsn); }) - .onErrorResume(throwable -> Mono.just(false)); + .onErrorResume(throwable -> { + + barrierRequest.requestContext.forceRefreshAddressCache = false; + + if (throwable instanceof CosmosException) { + CosmosException cosmosException = Utils.as(throwable, CosmosException.class); + + if (com.azure.cosmos.implementation.Exceptions.isAvoidQuorumSelectionException(cosmosException)) { + + bailFromWriteBarrierLoop.v = true; + cosmosExceptionValueHolder.v = cosmosException; + return Mono.just(false); + } + + bailFromWriteBarrierLoop.v = false; + return Mono.just(false); + } + + bailFromWriteBarrierLoop.v = false; + return Mono.just(false); + }); } void startBackgroundAddressRefresh(RxDocumentServiceRequest request) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java index a56363369682..5b700ee22fea 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java @@ -674,6 +674,7 @@ private Mono waitForReadBarrierAsync( } if (isAvoidQuorumSelectionStoreResult) { + readBarrierRetryCount.decrementAndGet(); return this.isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( barrierRequest, readBarrierLsn, @@ -756,6 +757,7 @@ private Mono waitForReadBarrierAsync( } if (isAvoidQuorumSelectionStoreResult) { + readBarrierRetryCountMultiRegion.getAndDecrement(); return this.isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( barrierRequest, readBarrierLsn, @@ -903,40 +905,50 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep long targetGlobalCommittedLSN, MutableVolatile cosmosExceptionValueHolder, MutableVolatile bailFromReadBarrierLoop, - CosmosException cosmosExceptionFromStoreResult) { + CosmosException cosmosExceptionInStoreResult) { return performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( barrierRequest, true, readBarrierLsn, - targetGlobalCommittedLSN).flatMap(barrierStatusFromPrimary -> { + targetGlobalCommittedLSN, + cosmosExceptionValueHolder, + bailFromReadBarrierLoop).flatMap(isBarrierFromPrimarySuccessful -> { - if (barrierStatusFromPrimary) { + if (isBarrierFromPrimarySuccessful) { bailFromReadBarrierLoop.v = true; cosmosExceptionValueHolder.v = null; return Mono.just(true); } + if (bailFromReadBarrierLoop.v) { bailFromReadBarrierLoop.v = true; cosmosExceptionValueHolder.v = Utils.createCosmosException( - HttpConstants.StatusCodes.SERVICE_UNAVAILABLE, - cosmosExceptionFromStoreResult.getSubStatusCode(), - cosmosExceptionFromStoreResult, + HttpConstants.StatusCodes.REQUEST_TIMEOUT, + cosmosExceptionInStoreResult.getSubStatusCode(), + cosmosExceptionInStoreResult, null); return Mono.just(false); + } else { + bailFromReadBarrierLoop.v = false; + cosmosExceptionValueHolder.v = null; + return Mono.empty(); + } }); } private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( - RxDocumentServiceRequest entity, + RxDocumentServiceRequest barrierRequest, boolean requiresValidLsn, long readBarrierLsn, - long targetGlobalCommittedLSN) { + long targetGlobalCommittedLSN, + MutableVolatile cosmosExceptionValueHolder, + MutableVolatile bailFromReadBarrierLoop) { - entity.requestContext.forceRefreshAddressCache = true; + barrierRequest.requestContext.forceRefreshAddressCache = true; Mono storeResultObs = this.storeReader.readPrimaryAsync( - entity, requiresValidLsn, false /*useSessionToken*/); + barrierRequest, requiresValidLsn, false /*useSessionToken*/); return storeResultObs.flatMap(storeResult -> { if (!storeResult.isValid) { @@ -949,7 +961,27 @@ private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied return Mono.just(hasRequiredLsn && hasRequiredGlobalCommittedLsn); }) - .onErrorResume(throwable -> Mono.just(false)); + .onErrorResume(throwable -> { + + barrierRequest.requestContext.forceRefreshAddressCache = false; + + if (throwable instanceof CosmosException) { + CosmosException cosmosException = Utils.as(throwable, CosmosException.class); + + if (com.azure.cosmos.implementation.Exceptions.isAvoidQuorumSelectionException(cosmosException)) { + + bailFromReadBarrierLoop.v = true; + cosmosExceptionValueHolder.v = cosmosException; + return Mono.just(false); + } + + bailFromReadBarrierLoop.v = false; + return Mono.just(false); + } + + bailFromReadBarrierLoop.v = false; + return Mono.just(false); + }); } private enum ReadQuorumResultKind { From 150f2d01e122bf862bc60e47e72feb58e14e7031 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 19 Nov 2025 12:58:58 -0500 Subject: [PATCH 19/30] Verify barrier bail out criteria. --- .../directconnectivity/QuorumReader.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java index 5b700ee22fea..8b5a58310cb0 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java @@ -915,12 +915,12 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep cosmosExceptionValueHolder, bailFromReadBarrierLoop).flatMap(isBarrierFromPrimarySuccessful -> { - if (isBarrierFromPrimarySuccessful) { - bailFromReadBarrierLoop.v = true; - cosmosExceptionValueHolder.v = null; + if (isBarrierFromPrimarySuccessful) { + bailFromReadBarrierLoop.v = true; + cosmosExceptionValueHolder.v = null; - return Mono.just(true); - } + return Mono.just(true); + } if (bailFromReadBarrierLoop.v) { bailFromReadBarrierLoop.v = true; @@ -935,7 +935,7 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep cosmosExceptionValueHolder.v = null; return Mono.empty(); } - }); + }); } private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( From 84deb3160cd89052e3aeca6c7684f388ea709ec5 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 19 Nov 2025 14:19:08 -0500 Subject: [PATCH 20/30] Managing merge. --- sdk/cosmos/azure-cosmos-tests/pom.xml | 22 ----------- ...InjectionServerErrorRuleOnDirectTests.java | 6 +-- .../com/azure/cosmos/rx/TestSuiteBase.java | 4 +- .../fault-injection-barrier-testng.xml | 38 ------------------- sdk/cosmos/live-platform-matrix.json | 11 ------ 5 files changed, 5 insertions(+), 76 deletions(-) delete mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/resources/fault-injection-barrier-testng.xml diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index 4ccc372334e7..96e54e9d0057 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -789,28 +789,6 @@ Licensed under the MIT License. - - - fault-injection-barrier - - fault-injection-barrier - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 3.5.3 - - - src/test/resources/fault-injection-barrier-testng.xml - - - - - - - multi-region-strong diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java index 58998a9ab6e4..08e1d7332eab 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/FaultInjectionServerErrorRuleOnDirectTests.java @@ -84,7 +84,7 @@ public FaultInjectionServerErrorRuleOnDirectTests(CosmosClientBuilder clientBuil this.subscriberValidationTimeout = TIMEOUT; } - @BeforeClass(groups = {"multi-region", "long", "fast", "fi-multi-master", "fault-injection-barrier"}, timeOut = TIMEOUT) + @BeforeClass(groups = {"multi-region", "long", "fast", "fi-multi-master", "multi-region-strong"}, timeOut = TIMEOUT) public void beforeClass() { clientWithoutPreferredRegions = getClientBuilder() .preferredRegions(new ArrayList<>()) @@ -1057,7 +1057,7 @@ public void faultInjectionServerErrorRuleTests_HitLimit() throws JsonProcessingE } } - @AfterClass(groups = {"multi-region", "long", "fast", "fi-multi-master", "fault-injection-barrier"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + @AfterClass(groups = {"multi-region", "long", "fast", "fi-multi-master", "multi-region-strong"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() { safeClose(clientWithoutPreferredRegions); } @@ -1475,7 +1475,7 @@ public void faultInjectionInjectTcpResponseDelay() throws JsonProcessingExceptio } } - @Test(groups = {"fault-injection-barrier"}, dataProvider = "barrierRequestServerErrorResponseProvider", timeOut = 2 * TIMEOUT) + @Test(groups = {"multi-region-strong"}, dataProvider = "barrierRequestServerErrorResponseProvider", timeOut = 2 * TIMEOUT) public void faultInjection_serverError_barrierRequest( OperationType operationType, FaultInjectionServerErrorType serverErrorType, diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index 1649852a3dde..e10d4b33932c 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -205,7 +205,7 @@ public CosmosAsyncDatabase getDatabase(String id) { @BeforeSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "emulator-vnext", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "fault-injection-barrier", "multi-region-strong"}, timeOut = SUITE_SETUP_TIMEOUT) + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong"}, timeOut = SUITE_SETUP_TIMEOUT) public void beforeSuite() { logger.info("beforeSuite Started"); @@ -230,7 +230,7 @@ public void parallelizeUnitTests(ITestContext context) { @AfterSuite(groups = {"thinclient", "fast", "long", "direct", "multi-region", "multi-master", "flaky-multi-master", "emulator", "split", "query", "cfp-split", "circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", - "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "fault-injection-barrier", "multi-region-strong"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) + "circuit-breaker-read-all-read-many", "fi-multi-master", "long-emulator", "fi-thinclient-multi-region", "fi-thinclient-multi-master", "multi-region-strong"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) public void afterSuite() { logger.info("afterSuite Started"); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fault-injection-barrier-testng.xml b/sdk/cosmos/azure-cosmos-tests/src/test/resources/fault-injection-barrier-testng.xml deleted file mode 100644 index 6444f7176010..000000000000 --- a/sdk/cosmos/azure-cosmos-tests/src/test/resources/fault-injection-barrier-testng.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index f5294bc1e402..e5771c249c1c 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -15,7 +15,6 @@ "-Pmulti-region": "MultiRegion", "-Pmulti-region-strong": "MultiRegionStrong", "-Plong": "Long", - "-Pfault-injection-barrier": "Fault-injection-barrier", "-DargLine=\"-Dazure.cosmos.directModeProtocol=Tcp\"": "TCP", "Session": "", "ubuntu": "", @@ -180,16 +179,6 @@ "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } } }, - { - "DESIRED_CONSISTENCIES": "[\"Strong\"]", - "ACCOUNT_CONSISTENCY": "Strong", - "PROTOCOLS": "[\"Tcp\"]", - "ProfileFlag": [ "-Pfault-injection-barrier" ], - "ArmTemplateParameters": "@{ enableMultipleWriteLocations = $false; defaultConsistencyLevel = 'Strong'; enableMultipleRegions = $true }", - "Agent": { - "ubuntu": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" } - } - }, { "DESIRED_CONSISTENCIES": "[\"Strong\"]", "ACCOUNT_CONSISTENCY": "Strong", From a49950b2449823b811d9e17236b579c67c2ab6a5 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 19 Nov 2025 15:39:16 -0500 Subject: [PATCH 21/30] Fix compilation errors. --- .../cosmos/implementation/directconnectivity/StoreResponse.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java index 52681e2ed6fa..f3034f0c1a44 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java @@ -202,7 +202,7 @@ public String getHeaderValue(String attribute) { } //NOTE: only used for testing purpose to change the response header value - private void setHeaderValue(String headerName, String value) { + public void setHeaderValue(String headerName, String value) { if (this.responseHeaderValues == null || this.responseHeaderNames.length != this.responseHeaderValues.length) { return; } From 822ee0f96105a96a21435f470d74d79d697dc6a2 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 19 Nov 2025 18:39:22 -0500 Subject: [PATCH 22/30] Fix tests. --- .../cosmos/BailOutFromBarrierE2ETests.java | 108 +++++------------- ...ortClientWithStoreResponseInterceptor.java | 72 ------------ .../directconnectivity/QuorumReader.java | 2 +- .../directconnectivity/StoreReader.java | 9 ++ 4 files changed, 37 insertions(+), 154 deletions(-) delete mode 100644 sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RntbdTransportClientWithStoreResponseInterceptor.java diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java index 33c04b84bdf6..fb426cc8351b 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java @@ -11,21 +11,15 @@ import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.ImplementationBridgeHelpers; import com.azure.cosmos.implementation.OperationType; -import com.azure.cosmos.implementation.RxDocumentClientImpl; import com.azure.cosmos.implementation.Strings; -import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.directconnectivity.ConsistencyReader; import com.azure.cosmos.implementation.directconnectivity.ConsistencyWriter; -import com.azure.cosmos.implementation.directconnectivity.ReflectionUtils; -import com.azure.cosmos.implementation.directconnectivity.ReplicatedResourceClient; -import com.azure.cosmos.implementation.directconnectivity.RntbdTransportClient; -import com.azure.cosmos.implementation.directconnectivity.StoreClient; -import com.azure.cosmos.implementation.directconnectivity.StoreReader; import com.azure.cosmos.implementation.directconnectivity.StoreResponseDiagnostics; import com.azure.cosmos.implementation.directconnectivity.StoreResultDiagnostics; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; import com.azure.cosmos.rx.TestSuiteBase; +import com.azure.cosmos.test.implementation.interceptor.CosmosInterceptorHelper; import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -258,9 +252,6 @@ public void validateHeadRequestLeaseNotFoundBailout( AtomicInteger successfulHeadCountTracker = new AtomicInteger(); AtomicInteger failedHeadCountTracker = new AtomicInteger(); - Utils.ValueHolder originalRntbdTransportClientHolder = new Utils.ValueHolder<>(); - - RntbdTransportClientWithStoreResponseInterceptor interceptorClient = createClientWithInterceptor(targetClient, originalRntbdTransportClientHolder); try { @@ -268,43 +259,36 @@ public void validateHeadRequestLeaseNotFoundBailout( if (enterPostQuorumSelectionOnlyBarrierLoop) { if (OperationType.Read.equals(operationTypeForWhichBarrierFlowIsTriggered)) { - interceptorClient - .setResponseInterceptor( - StoreResponseInterceptorUtils.forceSuccessfulBarriersOnReadUntilQuorumSelectionThenForceBarrierFailures( - effectiveConsistencyLevel, - this.regionNameToEndpoint.get(this.preferredRegions.get(0)), - successfulHeadRequestCountWhichDontMeetBarrier, - successfulHeadCountTracker, - headFailureCount, - failedHeadCountTracker, - HttpConstants.StatusCodes.GONE, - HttpConstants.SubStatusCodes.LEASE_NOT_FOUND - )); + CosmosInterceptorHelper.registerTransportClientInterceptor(targetClient, StoreResponseInterceptorUtils.forceSuccessfulBarriersOnReadUntilQuorumSelectionThenForceBarrierFailures( + effectiveConsistencyLevel, + this.regionNameToEndpoint.get(this.preferredRegions.get(0)), + successfulHeadRequestCountWhichDontMeetBarrier, + successfulHeadCountTracker, + headFailureCount, + failedHeadCountTracker, + HttpConstants.StatusCodes.GONE, + HttpConstants.SubStatusCodes.LEASE_NOT_FOUND + )); } } else { - if (OperationType.Create.equals(operationTypeForWhichBarrierFlowIsTriggered)) { - interceptorClient - .setResponseInterceptor( - StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( - effectiveConsistencyLevel, - this.regionNameToEndpoint.get(this.preferredRegions.get(1)), - headFailureCount, - failedHeadCountTracker, - HttpConstants.StatusCodes.GONE, - HttpConstants.SubStatusCodes.LEASE_NOT_FOUND - )); + CosmosInterceptorHelper.registerTransportClientInterceptor(targetClient, StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( + effectiveConsistencyLevel, + this.regionNameToEndpoint.get(this.preferredRegions.get(1)), + headFailureCount, + failedHeadCountTracker, + HttpConstants.StatusCodes.GONE, + HttpConstants.SubStatusCodes.LEASE_NOT_FOUND + )); } else if (OperationType.Read.equals(operationTypeForWhichBarrierFlowIsTriggered)) { - interceptorClient - .setResponseInterceptor( - StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( - effectiveConsistencyLevel, - this.regionNameToEndpoint.get(this.preferredRegions.get(0)), - headFailureCount, - failedHeadCountTracker, - HttpConstants.StatusCodes.GONE, - HttpConstants.SubStatusCodes.LEASE_NOT_FOUND - )); + CosmosInterceptorHelper.registerTransportClientInterceptor(targetClient, StoreResponseInterceptorUtils.forceBarrierFollowedByBarrierFailure( + effectiveConsistencyLevel, + this.regionNameToEndpoint.get(this.preferredRegions.get(0)), + headFailureCount, + failedHeadCountTracker, + HttpConstants.StatusCodes.GONE, + HttpConstants.SubStatusCodes.LEASE_NOT_FOUND + )); } } @@ -392,48 +376,10 @@ public void validateHeadRequestLeaseNotFoundBailout( } } finally { - - if (originalRntbdTransportClientHolder.v != null) { - originalRntbdTransportClientHolder.v.close(); - } - - interceptorClient.close(); safeClose(targetClient); } } - private RntbdTransportClientWithStoreResponseInterceptor createClientWithInterceptor( - CosmosAsyncClient targetClient, - Utils.ValueHolder originalRntbdTransportClientHolder) { - - // Get internal client - AsyncDocumentClient asyncDocumentClient = ReflectionUtils.getAsyncDocumentClient(targetClient); - RxDocumentClientImpl rxDocumentClient = (RxDocumentClientImpl) asyncDocumentClient; - - // Get store client and components - StoreClient storeClient = ReflectionUtils.getStoreClient(rxDocumentClient); - ReplicatedResourceClient replicatedResourceClient = ReflectionUtils.getReplicatedResourceClient(storeClient); - ConsistencyReader consistencyReader = ReflectionUtils.getConsistencyReader(replicatedResourceClient); - ConsistencyWriter consistencyWriter = ReflectionUtils.getConsistencyWriter(replicatedResourceClient); - StoreReader storeReaderFromConsistencyReader = ReflectionUtils.getStoreReader(consistencyReader); - StoreReader storeReaderFromConsistencyWriter = ReflectionUtils.getStoreReader(consistencyWriter); - - // Get the original transport client - RntbdTransportClient originalTransportClient = (RntbdTransportClient) ReflectionUtils.getTransportClient(replicatedResourceClient); - originalRntbdTransportClientHolder.v = originalTransportClient; - - // Create interceptor client - RntbdTransportClientWithStoreResponseInterceptor interceptorClient = - new RntbdTransportClientWithStoreResponseInterceptor(originalTransportClient); - - // Set the interceptor client on both reader and writer - ReflectionUtils.setTransportClient(storeReaderFromConsistencyReader, interceptorClient); - ReflectionUtils.setTransportClient(storeReaderFromConsistencyWriter, interceptorClient); - ReflectionUtils.setTransportClient(consistencyWriter, interceptorClient); - - return interceptorClient; - } - private void validateContactedRegions(CosmosDiagnostics diagnostics, int expectedRegionsContactedCount) { CosmosDiagnosticsContext cosmosDiagnosticsContext = diagnostics.getDiagnosticsContext(); diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RntbdTransportClientWithStoreResponseInterceptor.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RntbdTransportClientWithStoreResponseInterceptor.java deleted file mode 100644 index 5e64b209d189..000000000000 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/RntbdTransportClientWithStoreResponseInterceptor.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos; - -import com.azure.cosmos.implementation.GlobalEndpointManager; -import com.azure.cosmos.implementation.RxDocumentServiceRequest; -import com.azure.cosmos.implementation.directconnectivity.RntbdTransportClient; -import com.azure.cosmos.implementation.directconnectivity.StoreResponse; -import com.azure.cosmos.implementation.directconnectivity.Uri; -import com.azure.cosmos.implementation.directconnectivity.rntbd.ProactiveOpenConnectionsProcessor; -import com.azure.cosmos.implementation.faultinjection.IFaultInjectorProvider; -import com.azure.cosmos.models.CosmosContainerIdentity; -import reactor.core.publisher.Mono; - -import java.util.List; -import java.util.function.BiFunction; - -public class RntbdTransportClientWithStoreResponseInterceptor extends RntbdTransportClient { - private final RntbdTransportClient underlying; - private BiFunction responseInterceptor; - - public RntbdTransportClientWithStoreResponseInterceptor(RntbdTransportClient underlying) { - super(underlying); - this.underlying = underlying; - } - - public void setResponseInterceptor(BiFunction responseInterceptor) { - this.responseInterceptor = responseInterceptor; - } - - @Override - public Mono invokeStoreAsync(Uri physicalAddress, RxDocumentServiceRequest request) { - return this.underlying.invokeStoreAsync(physicalAddress, request) - .map(response -> { - if (responseInterceptor != null) { - return responseInterceptor.apply(request, response); - } - return response; - }); - } - - @Override - public void configureFaultInjectorProvider(IFaultInjectorProvider injectorProvider) { - this.underlying.configureFaultInjectorProvider(injectorProvider); - } - - @Override - public GlobalEndpointManager getGlobalEndpointManager() { - return this.underlying.getGlobalEndpointManager(); - } - - @Override - public ProactiveOpenConnectionsProcessor getProactiveOpenConnectionsProcessor() { - return this.underlying.getProactiveOpenConnectionsProcessor(); - } - - @Override - public void recordOpenConnectionsAndInitCachesCompleted(List cosmosContainerIdentities) { - this.underlying.recordOpenConnectionsAndInitCachesCompleted(cosmosContainerIdentities); - } - - @Override - public void recordOpenConnectionsAndInitCachesStarted(List cosmosContainerIdentities) { - this.underlying.recordOpenConnectionsAndInitCachesStarted(cosmosContainerIdentities); - } - - @Override - public void close() { - this.underlying.close(); - } -} diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java index 8b5a58310cb0..2d3d60db8da5 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java @@ -925,7 +925,7 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep if (bailFromReadBarrierLoop.v) { bailFromReadBarrierLoop.v = true; cosmosExceptionValueHolder.v = Utils.createCosmosException( - HttpConstants.StatusCodes.REQUEST_TIMEOUT, + HttpConstants.StatusCodes.SERVICE_UNAVAILABLE, cosmosExceptionInStoreResult.getSubStatusCode(), cosmosExceptionInStoreResult, null); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java index b3e17f406036..a6046151b23f 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreReader.java @@ -682,6 +682,15 @@ private Mono readPrimaryInternalAsync( true, primaryUriReference.get(), replicaStatusList); + + if (storeTaskException instanceof CosmosException) { + CosmosException cosmosException = (CosmosException) storeTaskException; + + if (com.azure.cosmos.implementation.Exceptions.isAvoidQuorumSelectionException(cosmosException)) { + return Mono.error(cosmosException); + } + } + return Mono.just(storeResult); } catch (CosmosException e) { // RxJava1 doesn't allow throwing checked exception from Observable operators From bca781b50c1cc5b9c886134d5694c924d2fb4fcd Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Wed, 19 Nov 2025 18:43:45 -0500 Subject: [PATCH 23/30] Fix tests. --- .../directconnectivity/RntbdTransportClient.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java index ac8f33cccb18..22c5ed8624b6 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/RntbdTransportClient.java @@ -145,18 +145,6 @@ public RntbdTransportClient( globalEndpointManager); } - public RntbdTransportClient(RntbdTransportClient other) { - this.serverErrorInjector = other.serverErrorInjector; - this.endpointProvider = other.endpointProvider; - this.id = instanceCount.incrementAndGet(); - this.tag = RntbdTransportClient.tag(this.id); - this.channelAcquisitionContextLatencyThresholdInMillis = other.channelAcquisitionContextLatencyThresholdInMillis; - this.globalEndpointManager = other.globalEndpointManager; - this.addressSelector = other.addressSelector; - this.proactiveOpenConnectionsProcessor = other.proactiveOpenConnectionsProcessor; - this.metricConfig = other.metricConfig; - } - RntbdTransportClient( final Options options, final SslContext sslContext, From 510e9c496a1b78082cfb085aba170f093ccfe9f7 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 20 Nov 2025 10:22:02 -0500 Subject: [PATCH 24/30] Addressing comments. --- .../java/com/azure/cosmos/BailOutFromBarrierE2ETests.java | 1 + .../StoreResponseInterceptorUtils.java | 8 ++++---- .../implementation/directconnectivity/StoreResponse.java | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) rename sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/{ => implementation/directconnectivity}/StoreResponseInterceptorUtils.java (95%) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java index fb426cc8351b..7b66f4012b9d 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java @@ -15,6 +15,7 @@ import com.azure.cosmos.implementation.directconnectivity.ConsistencyReader; import com.azure.cosmos.implementation.directconnectivity.ConsistencyWriter; import com.azure.cosmos.implementation.directconnectivity.StoreResponseDiagnostics; +import com.azure.cosmos.implementation.directconnectivity.StoreResponseInterceptorUtils; import com.azure.cosmos.implementation.directconnectivity.StoreResultDiagnostics; import com.azure.cosmos.models.CosmosItemResponse; import com.azure.cosmos.models.PartitionKey; diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/StoreResponseInterceptorUtils.java similarity index 95% rename from sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java rename to sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/StoreResponseInterceptorUtils.java index d8387f6e63db..bdf0fee5cf86 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/StoreResponseInterceptorUtils.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/StoreResponseInterceptorUtils.java @@ -1,13 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package com.azure.cosmos; +package com.azure.cosmos.implementation.directconnectivity; +import com.azure.cosmos.ConsistencyLevel; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.RxDocumentServiceRequest; import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.implementation.directconnectivity.StoreResponse; -import com.azure.cosmos.implementation.directconnectivity.WFConstants; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; @@ -79,7 +78,8 @@ public static BiFunction int maxAllowedFailureCount, AtomicInteger failureCount, int statusCode, - int subStatusCode) { + int subStatusCode) { + return (request, storeResponse) -> { if (OperationType.Read.equals(request.getOperationType()) && regionName.equals(request.requestContext.regionalRoutingContextToRoute.getGatewayRegionalEndpoint().toString())) { diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java index f3034f0c1a44..fb83e3cf26c2 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/StoreResponse.java @@ -202,7 +202,7 @@ public String getHeaderValue(String attribute) { } //NOTE: only used for testing purpose to change the response header value - public void setHeaderValue(String headerName, String value) { + void setHeaderValue(String headerName, String value) { if (this.responseHeaderValues == null || this.responseHeaderNames.length != this.responseHeaderValues.length) { return; } From 6fce95bc44ecb39f05b22e91acf9df1398a1b108 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Thu, 20 Nov 2025 10:29:52 -0500 Subject: [PATCH 25/30] Addressing comments. --- sdk/cosmos/azure-cosmos-tests/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sdk/cosmos/azure-cosmos-tests/pom.xml b/sdk/cosmos/azure-cosmos-tests/pom.xml index d77243f24ec0..3131458840f7 100644 --- a/sdk/cosmos/azure-cosmos-tests/pom.xml +++ b/sdk/cosmos/azure-cosmos-tests/pom.xml @@ -921,6 +921,12 @@ Licensed under the MIT License. src/test/resources/multi-region-strong.xml + + true + 1 + 256 + paranoid + From 73be72cd7e1f8c9927b2a60c71134a220832ac7c Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Sat, 22 Nov 2025 18:25:15 -0500 Subject: [PATCH 26/30] Addressing review comments. --- .../directconnectivity/ConsistencyWriter.java | 48 ++++++++-------- .../directconnectivity/QuorumReader.java | 56 ++++++++++--------- 2 files changed, 53 insertions(+), 51 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java index d820380b217f..4c0ad501a40b 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java @@ -46,6 +46,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -149,8 +150,7 @@ Mono writePrivateAsync( TimeoutHelper timeout, boolean forceRefresh) { - final MutableVolatile cosmosExceptionValueHolder = new MutableVolatile<>(null); - final MutableVolatile bailFromWriteBarrierLoopValueHolder = new MutableVolatile<>(false); + final AtomicReference cosmosExceptionValueHolder = new AtomicReference<>(null); if (timeout.isElapsed() && // skip throwing RequestTimeout on first retry because the first retry with @@ -305,8 +305,8 @@ Mono writePrivateAsync( if (!v) { logger.info("ConsistencyWriter: Write barrier has not been met for global strong request. SelectedGlobalCommittedLsn: {}", request.requestContext.globalCommittedSelectedLSN); - if (cosmosExceptionValueHolder.v != null) { - return Mono.error(cosmosExceptionValueHolder.v); + if (cosmosExceptionValueHolder.get() != null) { + return Mono.error(cosmosExceptionValueHolder.get()); } return Mono.error(new GoneException(RMResources.GlobalStrongWriteBarrierNotMet, @@ -335,7 +335,7 @@ boolean isGlobalStrongRequest(RxDocumentServiceRequest request, StoreResponse re Mono barrierForGlobalStrong( RxDocumentServiceRequest request, StoreResponse response, - MutableVolatile cosmosExceptionValueHolder) { + AtomicReference cosmosExceptionValueHolder) { try { if (ReplicatedResourceClient.isGlobalStrongEnabled() && this.isGlobalStrongRequest(request, response)) { @@ -374,8 +374,8 @@ Mono barrierForGlobalStrong( logger.error("ConsistencyWriter: Write barrier has not been met for global strong request. SelectedGlobalCommittedLsn: {}", request.requestContext.globalCommittedSelectedLSN); - if (cosmosExceptionValueHolder.v != null) { - return Mono.error(cosmosExceptionValueHolder.v); + if (cosmosExceptionValueHolder.get() != null) { + return Mono.error(cosmosExceptionValueHolder.get()); } // RxJava1 doesn't allow throwing checked exception @@ -404,7 +404,7 @@ Mono barrierForGlobalStrong( private Mono waitForWriteBarrierAsync( RxDocumentServiceRequest barrierRequest, long selectedGlobalCommittedLsn, - MutableVolatile cosmosExceptionValueHolder) { + AtomicReference cosmosExceptionValueHolder) { AtomicInteger writeBarrierRetryCount = new AtomicInteger(ConsistencyWriter.MAX_NUMBER_OF_WRITE_BARRIER_READ_RETRIES); AtomicLong maxGlobalCommittedLsnReceived = new AtomicLong(0); @@ -504,10 +504,10 @@ static void getLsnAndGlobalCommittedLsn(StoreResponse response, Utils.ValueHolde private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( RxDocumentServiceRequest barrierRequest, long selectedGlobalCommittedLsn, - MutableVolatile cosmosExceptionValueHolder, + AtomicReference cosmosExceptionValueHolder, CosmosException cosmosExceptionInStoreResult) { - MutableVolatile bailFromWriteBarrierLoop = new MutableVolatile<>(false); + AtomicBoolean bailFromWriteBarrierLoop = new AtomicBoolean(false); return performOptimisticBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( barrierRequest, @@ -516,23 +516,23 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep bailFromWriteBarrierLoop).flatMap(isBarrierFromPrimarySuccessful -> { if (isBarrierFromPrimarySuccessful) { - bailFromWriteBarrierLoop.v = true; - cosmosExceptionValueHolder.v = null; + bailFromWriteBarrierLoop.set(true); + cosmosExceptionValueHolder.set(null); return Mono.just(true); } - if (bailFromWriteBarrierLoop.v) { - bailFromWriteBarrierLoop.v = true; - cosmosExceptionValueHolder.v = Utils.createCosmosException( + if (bailFromWriteBarrierLoop.get()) { + bailFromWriteBarrierLoop.set(true); + cosmosExceptionValueHolder.set(Utils.createCosmosException( HttpConstants.StatusCodes.REQUEST_TIMEOUT, cosmosExceptionInStoreResult.getSubStatusCode(), cosmosExceptionInStoreResult, - null); + null)); return Mono.just(false); } else { - bailFromWriteBarrierLoop.v = false; - cosmosExceptionValueHolder.v = null; + bailFromWriteBarrierLoop.set(false); + cosmosExceptionValueHolder.set(null); return Mono.empty(); } }); @@ -541,8 +541,8 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep private Mono performOptimisticBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( RxDocumentServiceRequest barrierRequest, long selectedGlobalCommittedLSN, - MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailFromWriteBarrierLoop) { + AtomicReference cosmosExceptionValueHolder, + AtomicBoolean bailFromWriteBarrierLoop) { barrierRequest.requestContext.forceRefreshAddressCache = true; Mono storeResultObs = this.storeReader.readPrimaryAsync( @@ -567,16 +567,16 @@ private Mono performOptimisticBarrierOnPrimaryAndDetermineIfBarrierCanB if (com.azure.cosmos.implementation.Exceptions.isAvoidQuorumSelectionException(cosmosException)) { - bailFromWriteBarrierLoop.v = true; - cosmosExceptionValueHolder.v = cosmosException; + bailFromWriteBarrierLoop.set(true); + cosmosExceptionValueHolder.set(cosmosException); return Mono.just(false); } - bailFromWriteBarrierLoop.v = false; + bailFromWriteBarrierLoop.set(false); return Mono.just(false); } - bailFromWriteBarrierLoop.v = false; + bailFromWriteBarrierLoop.set(false); return Mono.just(false); }); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java index 2d3d60db8da5..90dee376e232 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java @@ -32,8 +32,10 @@ import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import static com.azure.cosmos.implementation.Utils.ValueHolder; @@ -135,8 +137,8 @@ public Mono readStrongAsync( final MutableVolatile shouldRetryOnSecondary = new MutableVolatile<>(false); final MutableVolatile hasPerformedReadFromPrimary = new MutableVolatile<>(false); final MutableVolatile includePrimary = new MutableVolatile<>(false); - final MutableVolatile cosmosExceptionValueHolder = new MutableVolatile<>(null); - final MutableVolatile bailOnBarrierValueHolder = new MutableVolatile<>(false); + final AtomicReference cosmosExceptionValueHolder = new AtomicReference<>(null); + final AtomicBoolean bailOnBarrierValueHolder = new AtomicBoolean(false); return Flux.defer( // the following will be repeated till the repeat().takeUntil(.) condition is satisfied. @@ -297,8 +299,8 @@ public Mono readStrongAsync( .switchIfEmpty(Flux.defer(() -> { logger.info("Could not complete read quorum with read quorum value of {}", readQuorumValue); - if (cosmosExceptionValueHolder.v != null) { - return Flux.error(cosmosExceptionValueHolder.v); + if (cosmosExceptionValueHolder.get() != null) { + return Flux.error(cosmosExceptionValueHolder.get()); } return Flux.error(new GoneException( @@ -316,8 +318,8 @@ private Mono readQuorumAsync( int readQuorum, boolean includePrimary, ReadMode readMode, - MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailOnBarrierValueHolder) { + AtomicReference cosmosExceptionValueHolder, + AtomicBoolean bailOnBarrierValueHolder) { if (entity.requestContext.timeoutHelper.isElapsed()) { return Mono.error(new GoneException()); @@ -344,7 +346,7 @@ private Mono readQuorumAsync( return waitForObs.flatMap( waitFor -> { - if (bailOnBarrierValueHolder.v && cosmosExceptionValueHolder.v != null) { + if (bailOnBarrierValueHolder.get() && cosmosExceptionValueHolder.get() != null) { return Mono.just(new ReadQuorumResult( entity.requestContext.requestChargeTracker, ReadQuorumResultKind.QuorumNotPossibleInCurrentRegion, @@ -352,7 +354,7 @@ private Mono readQuorumAsync( globalCommittedLSN, storeResult, storeResponses, - cosmosExceptionValueHolder.v)); + cosmosExceptionValueHolder.get())); } if (!waitFor) { @@ -642,8 +644,8 @@ private Mono waitForReadBarrierAsync( final long readBarrierLsn, final long targetGlobalCommittedLSN, ReadMode readMode, - MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailFromReadBarrierLoopValueHolder) { + AtomicReference cosmosExceptionValueHolder, + AtomicBoolean bailFromReadBarrierLoopValueHolder) { AtomicInteger readBarrierRetryCount = new AtomicInteger(maxNumberOfReadBarrierReadRetries); AtomicInteger readBarrierRetryCountMultiRegion = new AtomicInteger(maxBarrierRetriesForMultiRegion); @@ -726,7 +728,7 @@ private Mono waitForReadBarrierAsync( return Flux.just(true); } - if (bailFromReadBarrierLoopValueHolder.v) { + if (bailFromReadBarrierLoopValueHolder.get()) { return Flux.just(false); } @@ -903,8 +905,8 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep RxDocumentServiceRequest barrierRequest, long readBarrierLsn, long targetGlobalCommittedLSN, - MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailFromReadBarrierLoop, + AtomicReference cosmosExceptionValueHolder, + AtomicBoolean bailFromReadBarrierLoop, CosmosException cosmosExceptionInStoreResult) { return performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied( @@ -916,23 +918,23 @@ private Mono isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionExcep bailFromReadBarrierLoop).flatMap(isBarrierFromPrimarySuccessful -> { if (isBarrierFromPrimarySuccessful) { - bailFromReadBarrierLoop.v = true; - cosmosExceptionValueHolder.v = null; + bailFromReadBarrierLoop.set(true); + cosmosExceptionValueHolder.set(null); return Mono.just(true); } - if (bailFromReadBarrierLoop.v) { - bailFromReadBarrierLoop.v = true; - cosmosExceptionValueHolder.v = Utils.createCosmosException( + if (bailFromReadBarrierLoop.get()) { + bailFromReadBarrierLoop.set(true); + cosmosExceptionValueHolder.set(Utils.createCosmosException( HttpConstants.StatusCodes.SERVICE_UNAVAILABLE, cosmosExceptionInStoreResult.getSubStatusCode(), cosmosExceptionInStoreResult, - null); + null)); return Mono.just(false); } else { - bailFromReadBarrierLoop.v = false; - cosmosExceptionValueHolder.v = null; + bailFromReadBarrierLoop.set(false); + cosmosExceptionValueHolder.set(null); return Mono.empty(); } }); @@ -943,8 +945,8 @@ private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied boolean requiresValidLsn, long readBarrierLsn, long targetGlobalCommittedLSN, - MutableVolatile cosmosExceptionValueHolder, - MutableVolatile bailFromReadBarrierLoop) { + AtomicReference cosmosExceptionValueHolder, + AtomicBoolean bailFromReadBarrierLoop) { barrierRequest.requestContext.forceRefreshAddressCache = true; Mono storeResultObs = this.storeReader.readPrimaryAsync( @@ -970,16 +972,16 @@ private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied if (com.azure.cosmos.implementation.Exceptions.isAvoidQuorumSelectionException(cosmosException)) { - bailFromReadBarrierLoop.v = true; - cosmosExceptionValueHolder.v = cosmosException; + bailFromReadBarrierLoop.set(true); + cosmosExceptionValueHolder.set(cosmosException); return Mono.just(false); } - bailFromReadBarrierLoop.v = false; + bailFromReadBarrierLoop.set(false); return Mono.just(false); } - bailFromReadBarrierLoop.v = false; + bailFromReadBarrierLoop.set(false); return Mono.just(false); }); } From 2729916efdfad53a4e7aa7b93c7160538ceb9805 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 24 Nov 2025 12:45:12 -0500 Subject: [PATCH 27/30] Refactoring. --- .../implementation/directconnectivity/ConsistencyWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java index 4c0ad501a40b..22af52c628e9 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java @@ -17,7 +17,6 @@ import com.azure.cosmos.implementation.ISessionContainer; import com.azure.cosmos.implementation.Integers; import com.azure.cosmos.implementation.InternalServerErrorException; -import com.azure.cosmos.implementation.MutableVolatile; import com.azure.cosmos.implementation.OperationType; import com.azure.cosmos.implementation.RMResources; import com.azure.cosmos.implementation.RequestChargeTracker; From 73cfe16bc4fe7f87b2911e7493f4ba70c6c791ba Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 24 Nov 2025 16:40:35 -0500 Subject: [PATCH 28/30] Addressing review comments. --- .../directconnectivity/ConsistencyWriter.java | 10 ++++++++-- .../directconnectivity/QuorumReader.java | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java index 22af52c628e9..f624c4606387 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java @@ -51,6 +51,8 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static com.azure.cosmos.implementation.Exceptions.isAvoidQuorumSelectionException; + /* * ConsistencyWriter has two modes for writing - local quorum-acked write and globally strong write. * @@ -436,7 +438,11 @@ private Mono waitForWriteBarrierAsync( } if (isAvoidQuorumSelectionStoreResult) { - writeBarrierRetryCount.getAndIncrement(); + + if (writeBarrierRetryCount.getAndIncrement() == 0) { + return Mono.just(false); + } + return this.isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( barrierRequest, selectedGlobalCommittedLsn, @@ -564,7 +570,7 @@ private Mono performOptimisticBarrierOnPrimaryAndDetermineIfBarrierCanB if (throwable instanceof CosmosException) { CosmosException cosmosException = Utils.as(throwable, CosmosException.class); - if (com.azure.cosmos.implementation.Exceptions.isAvoidQuorumSelectionException(cosmosException)) { + if (isAvoidQuorumSelectionException(cosmosException)) { bailFromWriteBarrierLoop.set(true); cosmosExceptionValueHolder.set(cosmosException); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java index 90dee376e232..0b7aaf02f7b2 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java @@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static com.azure.cosmos.implementation.Exceptions.isAvoidQuorumSelectionException; import static com.azure.cosmos.implementation.Utils.ValueHolder; // @@ -970,7 +971,7 @@ private Mono performBarrierOnPrimaryAndDetermineIfBarrierCanBeSatisfied if (throwable instanceof CosmosException) { CosmosException cosmosException = Utils.as(throwable, CosmosException.class); - if (com.azure.cosmos.implementation.Exceptions.isAvoidQuorumSelectionException(cosmosException)) { + if (isAvoidQuorumSelectionException(cosmosException)) { bailFromReadBarrierLoop.set(true); cosmosExceptionValueHolder.set(cosmosException); From abe4a76e28725f9db8634be7fbe0d102c0c139a1 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 24 Nov 2025 17:19:06 -0500 Subject: [PATCH 29/30] Addressing review comments. --- .../directconnectivity/ConsistencyWriter.java | 10 +++++----- .../directconnectivity/QuorumReader.java | 8 ++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java index f624c4606387..86b3e6cd0346 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/ConsistencyWriter.java @@ -414,6 +414,10 @@ private Mono waitForWriteBarrierAsync( return Flux.error(new RequestTimeoutException()); } + if (writeBarrierRetryCount.get() == 0) { + return Mono.just(false); + } + Mono> storeResultListObs = this.storeReader.readMultipleReplicaAsync( barrierRequest, true /*allowPrimary*/, @@ -438,11 +442,7 @@ private Mono waitForWriteBarrierAsync( } if (isAvoidQuorumSelectionStoreResult) { - - if (writeBarrierRetryCount.getAndIncrement() == 0) { - return Mono.just(false); - } - + writeBarrierRetryCount.decrementAndGet(); return this.isBarrierMeetPossibleInPresenceOfAvoidQuorumSelectionException( barrierRequest, selectedGlobalCommittedLsn, diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java index 0b7aaf02f7b2..487d2184db39 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/QuorumReader.java @@ -668,6 +668,10 @@ private Mono waitForReadBarrierAsync( boolean isAvoidQuorumSelectionStoreResult = false; CosmosException cosmosExceptionFromStoreResult = null; + if (readBarrierRetryCount.get() == 0) { + return Mono.just(false); + } + for (StoreResult storeResult : responses) { if (storeResult.isAvoidQuorumSelectionException) { isAvoidQuorumSelectionStoreResult = true; @@ -741,6 +745,10 @@ private Mono waitForReadBarrierAsync( return Flux.error(new GoneException()); } + if (readBarrierRetryCountMultiRegion.get() == 0) { + return Flux.just(false); + } + Mono> responsesObs = this.storeReader.readMultipleReplicaAsync( barrierRequest, allowPrimary, readQuorum, true /*required valid LSN*/, false /*useSessionToken*/, readMode, false /*checkMinLSN*/, true /*forceReadAll*/); From d2a625c26b1875900444eeca92b1284c9c80f2d9 Mon Sep 17 00:00:00 2001 From: Abhijeet Mohanty Date: Mon, 24 Nov 2025 18:11:11 -0500 Subject: [PATCH 30/30] Addressing review comments. --- .../com/azure/cosmos/BailOutFromBarrierE2ETests.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java index 7b66f4012b9d..92ced6c37405 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/BailOutFromBarrierE2ETests.java @@ -134,11 +134,11 @@ public static Object[][] headRequestLeaseNotFoundScenarios() { { 2, OperationType.Read, true, 18, false, ConsistencyLevel.STRONG }, { 3, OperationType.Read, true, 18, false, ConsistencyLevel.STRONG }, { 4, OperationType.Read, true, 18, true, ConsistencyLevel.STRONG }, - { 1, OperationType.Read, true, 111, false, ConsistencyLevel.STRONG }, - { 2, OperationType.Read, true, 111, false, ConsistencyLevel.STRONG }, - { 3, OperationType.Read, true, 111, false, ConsistencyLevel.STRONG }, - { 4, OperationType.Read, true, 111, false, ConsistencyLevel.STRONG }, - { 5, OperationType.Read, true, 111, true, ConsistencyLevel.STRONG } + { 1, OperationType.Read, true, 108, false, ConsistencyLevel.STRONG }, + { 2, OperationType.Read, true, 108, false, ConsistencyLevel.STRONG }, + { 3, OperationType.Read, true, 108, false, ConsistencyLevel.STRONG }, + { 4, OperationType.Read, true, 108, false, ConsistencyLevel.STRONG }, + { 5, OperationType.Read, true, 108, true, ConsistencyLevel.STRONG } }; }