From 1069e252e71b042d061d5c02a08618810b21e2af Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov Date: Thu, 18 Dec 2025 16:57:47 -0800 Subject: [PATCH 1/5] CSHARP-5712: withTransaction API retries too frequently --- .../Core/Operations/RetryabilityHelper.cs | 12 ++++++ .../Operations/RetryabilityHelperTests.cs | 37 ++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs b/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs index 0ccde3f9c7d..d4fb10d683a 100644 --- a/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs +++ b/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs @@ -30,6 +30,7 @@ internal static class RetryabilityHelper private const string RetryableWriteErrorLabel = "RetryableWriteError"; // private static fields + private static readonly Random __backoffRandom = new Random(); private static readonly HashSet __resumableChangeStreamErrorCodes; private static readonly HashSet __resumableChangeStreamExceptions; private static readonly HashSet __retryableReadExceptions; @@ -108,6 +109,17 @@ public static void AddRetryableWriteErrorLabelIfRequired(MongoException exceptio } } + public static int GetRetryDelayMs(int attempt, double backoffBase = 2, int backoffInitial = 100, int backoffMax = 10_000) + { + Ensure.IsGreaterThanZero(attempt, nameof(attempt)); + Ensure.IsGreaterThanZero(backoffBase, nameof(backoffBase)); + Ensure.IsGreaterThanZero(backoffInitial, nameof(backoffInitial)); + Ensure.IsGreaterThan(backoffMax, backoffInitial, nameof(backoffMax)); + + var j = __backoffRandom.NextDouble(); + return (int)(j * Math.Min(backoffMax, backoffInitial * Math.Pow(backoffBase, attempt - 1))); + } + public static bool IsCommandRetryable(BsonDocument command) { return diff --git a/tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs b/tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs index 9fc9db65be4..05a36b774b5 100644 --- a/tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs +++ b/tests/MongoDB.Driver.Tests/Core/Operations/RetryabilityHelperTests.cs @@ -21,7 +21,6 @@ using MongoDB.Driver.Core.Misc; using MongoDB.Driver.Core.TestHelpers; using Xunit; -using MongoDB.Driver.Core.Connections; namespace MongoDB.Driver.Core.Operations { @@ -102,6 +101,42 @@ public void AddRetryableWriteErrorLabelIfRequired_should_add_RetryableWriteError hasRetryableWriteErrorLabel.Should().Be(shouldAddErrorLabel); } + [Theory] + [InlineData(1, 2, 100, 10000, 0, 100)] + [InlineData(2, 2, 100, 10000, 0, 200)] + [InlineData(3, 2, 100, 10000, 0, 400)] + [InlineData(9999, 2, 100, 10000, 0, 10000)] + + [InlineData(1, 1.5, 100, 10000, 0, 100)] + [InlineData(2, 1.5, 100, 10000, 0, 150)] + [InlineData(3, 1.5, 100, 10000, 0, 225)] + [InlineData(9999, 1.5, 100, 10000, 0, 10000)] + public void GetRetryDelayMs_should_return_expected_results(int attempt, double backoffBase, int backoffInitial, int backoffMax, int expectedRangeMin, int expectedRangeMax) + { + var result = RetryabilityHelper.GetRetryDelayMs(attempt, backoffBase, backoffInitial, backoffMax); + + result.Should().BeInRange(expectedRangeMin, expectedRangeMax); + } + + [Theory] + [InlineData(-1, 2, 100, 1000, "attempt")] + [InlineData(0, 2, 100, 1000, "attempt")] + [InlineData(1, -1, 100, 1000, "backoffBase")] + [InlineData(1, 0, 100, 1000, "backoffBase")] + [InlineData(1, 2, -1, 1000, "backoffInitial")] + [InlineData(1, 2, 0, 1000, "backoffInitial")] + [InlineData(1, 2, 100, -1, "backoffMax")] + [InlineData(1, 2, 100, 0, "backoffMax")] + [InlineData(1, 2, 100, 50, "backoffMax")] + + public void GetRetryDelayMs_throws_on_wrong_parameters(int attempt, double backoffBase, int backoffInitial, int backoffMax, string expectedParameterName) + { + var exception = Record.Exception(() => RetryabilityHelper.GetRetryDelayMs(attempt, backoffBase, backoffInitial, backoffMax)); + + exception.Should().BeOfType().Subject + .ParamName.Should().Be(expectedParameterName); + } + [Theory] [InlineData("{ txnNumber : 1 }", true)] [InlineData("{ commitTransaction : 1 }", true)] From 9577f33347f4875d9aab6afc00e980c67cb79433 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov Date: Thu, 18 Dec 2025 17:11:11 -0800 Subject: [PATCH 2/5] pr --- src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs b/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs index d4fb10d683a..8a749925e85 100644 --- a/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs +++ b/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs @@ -30,7 +30,6 @@ internal static class RetryabilityHelper private const string RetryableWriteErrorLabel = "RetryableWriteError"; // private static fields - private static readonly Random __backoffRandom = new Random(); private static readonly HashSet __resumableChangeStreamErrorCodes; private static readonly HashSet __resumableChangeStreamExceptions; private static readonly HashSet __retryableReadExceptions; @@ -116,7 +115,11 @@ public static int GetRetryDelayMs(int attempt, double backoffBase = 2, int backo Ensure.IsGreaterThanZero(backoffInitial, nameof(backoffInitial)); Ensure.IsGreaterThan(backoffMax, backoffInitial, nameof(backoffMax)); - var j = __backoffRandom.NextDouble(); +#if NET6_0_OR_GREATER + var j = Random.Shared.NextDouble(); +#else + var j = (new Random()).NextDouble(); +#endif return (int)(j * Math.Min(backoffMax, backoffInitial * Math.Pow(backoffBase, attempt - 1))); } From c2cef3b2e599f1372e390c2b0fc04445dd2bf8c7 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov Date: Fri, 19 Dec 2025 15:19:45 -0800 Subject: [PATCH 3/5] CSHARP-5712: withTransaction API retries too frequently --- src/MongoDB.Driver/TransactionExecutor.cs | 174 ++++++++---------- .../ClientSessionHandleTests.cs | 1 + 2 files changed, 79 insertions(+), 96 deletions(-) diff --git a/src/MongoDB.Driver/TransactionExecutor.cs b/src/MongoDB.Driver/TransactionExecutor.cs index 32f94861a2e..e8f59b957ca 100644 --- a/src/MongoDB.Driver/TransactionExecutor.cs +++ b/src/MongoDB.Driver/TransactionExecutor.cs @@ -18,6 +18,7 @@ using System.Threading.Tasks; using MongoDB.Driver.Core.Bindings; using MongoDB.Driver.Core.Misc; +using MongoDB.Driver.Core.Operations; namespace MongoDB.Driver { @@ -35,28 +36,41 @@ public static TResult ExecuteWithRetries( IClock clock, CancellationToken cancellationToken) { + var attempt = 0; var transactionTimeout = transactionOptions?.Timeout ?? clientSession.Options.DefaultTransactionOptions?.Timeout; using var operationContext = new OperationContext(clock, transactionTimeout, cancellationToken); while (true) { + attempt++; clientSession.StartTransaction(transactionOptions); clientSession.WrappedCoreSession.CurrentTransaction.OperationContext = operationContext; - var callbackOutcome = ExecuteCallback(operationContext, clientSession, callback, cancellationToken); - if (callbackOutcome.ShouldRetryTransaction) + try { - continue; + var result = ExecuteCallback(operationContext, clientSession, callback, cancellationToken); + // Transaction could be completed by user's code inside the callback, skipping commit in such case. + if (IsTransactionInStartingOrInProgressState(clientSession)) + { + CommitWithRetries(operationContext, clientSession, cancellationToken); + } + + return result; } - if (!IsTransactionInStartingOrInProgressState(clientSession)) + catch (Exception ex) { - return callbackOutcome.Result; // assume callback intentionally ended the transaction - } + if (!HasErrorLabel(ex, TransientTransactionErrorLabel)) + { + throw; + } - var transactionHasBeenCommitted = CommitWithRetries(operationContext, clientSession, cancellationToken); - if (transactionHasBeenCommitted) - { - return callbackOutcome.Result; + var delay = GetRetryDelay(attempt); + if (HasTimedOut(operationContext, delay)) + { + throw; + } + + Thread.Sleep(delay); } } } @@ -68,97 +82,99 @@ public static async Task ExecuteWithRetriesAsync( IClock clock, CancellationToken cancellationToken) { - TimeSpan? transactionTimeout = transactionOptions?.Timeout ?? clientSession.Options.DefaultTransactionOptions?.Timeout; + var attempt = 0; + var transactionTimeout = transactionOptions?.Timeout ?? clientSession.Options.DefaultTransactionOptions?.Timeout; using var operationContext = new OperationContext(clock, transactionTimeout, cancellationToken); while (true) { + attempt++; clientSession.StartTransaction(transactionOptions); clientSession.WrappedCoreSession.CurrentTransaction.OperationContext = operationContext; - var callbackOutcome = await ExecuteCallbackAsync(operationContext, clientSession, callbackAsync, cancellationToken).ConfigureAwait(false); - if (callbackOutcome.ShouldRetryTransaction) + try { - continue; + var result = await ExecuteCallbackAsync(operationContext, clientSession, callbackAsync, cancellationToken).ConfigureAwait(false); + // Transaction could be completed by user's code inside the callback, skipping commit in such case. + if (IsTransactionInStartingOrInProgressState(clientSession)) + { + await CommitWithRetriesAsync(operationContext, clientSession, cancellationToken).ConfigureAwait(false); + } + + return result; } - if (!IsTransactionInStartingOrInProgressState(clientSession)) + catch (Exception ex) { - return callbackOutcome.Result; // assume callback intentionally ended the transaction - } + if (!HasErrorLabel(ex, TransientTransactionErrorLabel)) + { + throw; + } - var transactionHasBeenCommitted = await CommitWithRetriesAsync(operationContext, clientSession, cancellationToken).ConfigureAwait(false); - if (transactionHasBeenCommitted) - { - return callbackOutcome.Result; + var delay = GetRetryDelay(attempt); + if (HasTimedOut(operationContext, delay)) + { + throw; + } + + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); } } } - private static bool HasTimedOut(OperationContext operationContext) + private static TimeSpan GetRetryDelay(int attempt) + => TimeSpan.FromMilliseconds(RetryabilityHelper.GetRetryDelayMs(attempt, 1.5, 5, 500)); + + private static bool HasTimedOut(OperationContext operationContext, TimeSpan delay = default) { - return operationContext.IsTimedOut() || - (operationContext.RootContext.Timeout == null && operationContext.RootContext.Elapsed > __transactionTimeout); + if (operationContext.Timeout.HasValue) + { + return operationContext.Elapsed + delay >= operationContext.Timeout; + } + + return operationContext.RootContext.Elapsed + delay > __transactionTimeout; } - private static CallbackOutcome ExecuteCallback(OperationContext operationContext, IClientSessionHandle clientSession, Func callback, CancellationToken cancellationToken) + private static TResult ExecuteCallback(OperationContext operationContext, IClientSessionHandle clientSession, Func callback, CancellationToken cancellationToken) { try { - var result = callback(clientSession, cancellationToken); - return new CallbackOutcome.WithResult(result); + return callback(clientSession, cancellationToken); } - catch (Exception ex) + catch (Exception) when (IsTransactionInStartingOrInProgressState(clientSession)) { - if (IsTransactionInStartingOrInProgressState(clientSession)) + AbortTransactionOptions abortOptions = null; + if (operationContext.IsRootContextTimeoutConfigured()) { - AbortTransactionOptions abortOptions = null; - if (operationContext.IsRootContextTimeoutConfigured()) - { - abortOptions = new AbortTransactionOptions(operationContext.RootContext.Timeout); - } - - clientSession.AbortTransaction(abortOptions, cancellationToken); + abortOptions = new AbortTransactionOptions(operationContext.RootContext.Timeout); } - if (HasErrorLabel(ex, TransientTransactionErrorLabel) && !HasTimedOut(operationContext)) - { - return new CallbackOutcome.WithShouldRetryTransaction(); - } + clientSession.AbortTransaction(abortOptions, cancellationToken); throw; } } - private static async Task> ExecuteCallbackAsync(OperationContext operationContext, IClientSessionHandle clientSession, Func> callbackAsync, CancellationToken cancellationToken) + private static async Task ExecuteCallbackAsync(OperationContext operationContext, IClientSessionHandle clientSession, Func> callbackAsync, CancellationToken cancellationToken) { try { - var result = await callbackAsync(clientSession, cancellationToken).ConfigureAwait(false); - return new CallbackOutcome.WithResult(result); + return await callbackAsync(clientSession, cancellationToken).ConfigureAwait(false); } - catch (Exception ex) + catch (Exception) when (IsTransactionInStartingOrInProgressState(clientSession)) { - if (IsTransactionInStartingOrInProgressState(clientSession)) + AbortTransactionOptions abortOptions = null; + if (operationContext.IsRootContextTimeoutConfigured()) { - AbortTransactionOptions abortOptions = null; - if (operationContext.IsRootContextTimeoutConfigured()) - { - abortOptions = new AbortTransactionOptions(operationContext.RootContext.Timeout); - } - - await clientSession.AbortTransactionAsync(abortOptions, cancellationToken).ConfigureAwait(false); + abortOptions = new AbortTransactionOptions(operationContext.RootContext.Timeout); } - if (HasErrorLabel(ex, TransientTransactionErrorLabel) && !HasTimedOut(operationContext)) - { - return new CallbackOutcome.WithShouldRetryTransaction(); - } + await clientSession.AbortTransactionAsync(abortOptions, cancellationToken).ConfigureAwait(false); throw; } } - private static bool CommitWithRetries(OperationContext operationContext, IClientSessionHandle clientSession, CancellationToken cancellationToken) + private static void CommitWithRetries(OperationContext operationContext, IClientSessionHandle clientSession, CancellationToken cancellationToken) { while (true) { @@ -171,7 +187,7 @@ private static bool CommitWithRetries(OperationContext operationContext, IClient } clientSession.CommitTransaction(commitOptions, cancellationToken); - return true; + return; } catch (Exception ex) { @@ -180,17 +196,12 @@ private static bool CommitWithRetries(OperationContext operationContext, IClient continue; } - if (HasErrorLabel(ex, TransientTransactionErrorLabel) && !HasTimedOut(operationContext)) - { - return false; // the transaction will be retried - } - throw; } } } - private static async Task CommitWithRetriesAsync(OperationContext operationContext, IClientSessionHandle clientSession, CancellationToken cancellationToken) + private static async Task CommitWithRetriesAsync(OperationContext operationContext, IClientSessionHandle clientSession, CancellationToken cancellationToken) { while (true) { @@ -203,7 +214,7 @@ private static async Task CommitWithRetriesAsync(OperationContext operatio } await clientSession.CommitTransactionAsync(commitOptions, cancellationToken).ConfigureAwait(false); - return true; + return; } catch (Exception ex) { @@ -212,11 +223,6 @@ private static async Task CommitWithRetriesAsync(OperationContext operatio continue; } - if (HasErrorLabel(ex, TransientTransactionErrorLabel) && !HasTimedOut(operationContext)) - { - return false; // the transaction will be retried - } - throw; } } @@ -228,10 +234,8 @@ private static bool HasErrorLabel(Exception ex, string errorLabel) { return mongoException.HasErrorLabel(errorLabel); } - else - { - return false; - } + + return false; } private static bool IsMaxTimeMSExpiredException(Exception ex) @@ -279,27 +283,5 @@ private static bool ShouldRetryCommit(OperationContext operationContext, Excepti !HasTimedOut(operationContext) && !IsMaxTimeMSExpiredException(ex); } - - // nested types - internal abstract class CallbackOutcome - { - public virtual TResult Result => throw new InvalidOperationException(); - public virtual bool ShouldRetryTransaction => false; - - public class WithResult : CallbackOutcome - { - public WithResult(TResult result) - { - Result = result; - } - - public override TResult Result { get; } - } - - public class WithShouldRetryTransaction : CallbackOutcome - { - public override bool ShouldRetryTransaction => true; - } - } } } diff --git a/tests/MongoDB.Driver.Tests/ClientSessionHandleTests.cs b/tests/MongoDB.Driver.Tests/ClientSessionHandleTests.cs index 23021893994..7e5c74c99ee 100644 --- a/tests/MongoDB.Driver.Tests/ClientSessionHandleTests.cs +++ b/tests/MongoDB.Driver.Tests/ClientSessionHandleTests.cs @@ -533,6 +533,7 @@ public void WithTransaction_commit_after_callback_processing_should_be_processed } var subject = CreateSubject(coreSession: mockCoreSession.Object, clock: mockClock.Object); + SetupTransactionState(subject, true); if (async) { From c5c3dcdf0429ec408ab59a702fb19845e2ee9d77 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov Date: Fri, 19 Dec 2025 15:56:38 -0800 Subject: [PATCH 4/5] Pr --- src/MongoDB.Driver/Core/Misc/ThreadStaticRandom.cs | 11 +++++++++++ .../Core/Operations/RetryabilityHelper.cs | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/MongoDB.Driver/Core/Misc/ThreadStaticRandom.cs b/src/MongoDB.Driver/Core/Misc/ThreadStaticRandom.cs index 2c42f12d1e9..1c6037202ee 100644 --- a/src/MongoDB.Driver/Core/Misc/ThreadStaticRandom.cs +++ b/src/MongoDB.Driver/Core/Misc/ThreadStaticRandom.cs @@ -34,5 +34,16 @@ public static int Next(int maxValue) return random.Next(maxValue); } + + public static double NextDouble() + { + var random = __threadStaticRandom; + if (random == null) + { + random = __threadStaticRandom = new Random(); + } + + return random.NextDouble(); + } } } diff --git a/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs b/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs index 8a749925e85..f0673104345 100644 --- a/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs +++ b/src/MongoDB.Driver/Core/Operations/RetryabilityHelper.cs @@ -118,7 +118,7 @@ public static int GetRetryDelayMs(int attempt, double backoffBase = 2, int backo #if NET6_0_OR_GREATER var j = Random.Shared.NextDouble(); #else - var j = (new Random()).NextDouble(); + var j = ThreadStaticRandom.NextDouble(); #endif return (int)(j * Math.Min(backoffMax, backoffInitial * Math.Pow(backoffBase, attempt - 1))); } From fcc6211189188f78998353822962a1c6d4fbaa0f Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov Date: Fri, 19 Dec 2025 16:01:30 -0800 Subject: [PATCH 5/5] pr --- src/MongoDB.Driver/TransactionExecutor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MongoDB.Driver/TransactionExecutor.cs b/src/MongoDB.Driver/TransactionExecutor.cs index e8f59b957ca..a2eef7541f4 100644 --- a/src/MongoDB.Driver/TransactionExecutor.cs +++ b/src/MongoDB.Driver/TransactionExecutor.cs @@ -131,7 +131,7 @@ private static bool HasTimedOut(OperationContext operationContext, TimeSpan dela return operationContext.Elapsed + delay >= operationContext.Timeout; } - return operationContext.RootContext.Elapsed + delay > __transactionTimeout; + return operationContext.RootContext.Elapsed + delay >= __transactionTimeout; } private static TResult ExecuteCallback(OperationContext operationContext, IClientSessionHandle clientSession, Func callback, CancellationToken cancellationToken)