From 1d7a8078b71d8ff15e1c4ff992dbf0a4d061febe Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Fri, 21 Jul 2023 09:58:49 +0200 Subject: [PATCH 1/2] Retries with jitter --- src/Polly.Core/PublicAPI.Unshipped.txt | 5 +- src/Polly.Core/Retry/RetryBackoffType.cs | 13 ---- src/Polly.Core/Retry/RetryHelper.cs | 19 ++++- .../Retry/RetryResilienceStrategy.cs | 5 +- .../Retry/RetryStrategyOptions.TResult.cs | 15 +++- .../Retry/RetryHelperTests.cs | 77 ++++++++++++------- ...esilienceStrategyBuilderExtensionsTests.cs | 2 +- .../IssuesTests.StrategiesPerEndpoint_1365.cs | 3 +- 8 files changed, 89 insertions(+), 50 deletions(-) diff --git a/src/Polly.Core/PublicAPI.Unshipped.txt b/src/Polly.Core/PublicAPI.Unshipped.txt index 31833b0d5b3..6c8abfc48a4 100644 --- a/src/Polly.Core/PublicAPI.Unshipped.txt +++ b/src/Polly.Core/PublicAPI.Unshipped.txt @@ -319,9 +319,8 @@ Polly.Retry.OnRetryArguments.RetryDelay.get -> System.TimeSpan Polly.Retry.OnRetryArguments.RetryDelay.init -> void Polly.Retry.RetryBackoffType Polly.Retry.RetryBackoffType.Constant = 0 -> Polly.Retry.RetryBackoffType -Polly.Retry.RetryBackoffType.Exponential = 2 -> Polly.Retry.RetryBackoffType -Polly.Retry.RetryBackoffType.ExponentialWithJitter = 3 -> Polly.Retry.RetryBackoffType Polly.Retry.RetryBackoffType.Linear = 1 -> Polly.Retry.RetryBackoffType +Polly.Retry.RetryBackoffType.Exponential = 2 -> Polly.Retry.RetryBackoffType Polly.Retry.RetryDelayArguments Polly.Retry.RetryDelayArguments.Attempt.get -> int Polly.Retry.RetryDelayArguments.Attempt.init -> void @@ -350,6 +349,8 @@ Polly.Retry.RetryStrategyOptions.RetryDelayGenerator.set -> void Polly.Retry.RetryStrategyOptions.RetryStrategyOptions() -> void Polly.Retry.RetryStrategyOptions.ShouldHandle.get -> System.Func, System.Threading.Tasks.ValueTask>! Polly.Retry.RetryStrategyOptions.ShouldHandle.set -> void +Polly.Retry.RetryStrategyOptions.UseJitter.get -> bool +Polly.Retry.RetryStrategyOptions.UseJitter.set -> void Polly.RetryResilienceStrategyBuilderExtensions Polly.Telemetry.ExecutionAttemptArguments Polly.Telemetry.ExecutionAttemptArguments.Attempt.get -> int diff --git a/src/Polly.Core/Retry/RetryBackoffType.cs b/src/Polly.Core/Retry/RetryBackoffType.cs index 728af427c03..9e036121966 100644 --- a/src/Polly.Core/Retry/RetryBackoffType.cs +++ b/src/Polly.Core/Retry/RetryBackoffType.cs @@ -38,17 +38,4 @@ public enum RetryBackoffType /// 200ms, 400ms, 800ms. /// Exponential, - - /// - /// Exponential delay with randomization retry type, - /// making sure to mitigate any correlations. - /// - /// - /// 850ms, 1455ms, 3060ms. - /// - /// - /// In transient failures handling scenarios, this is the - /// recommended retry type. - /// - ExponentialWithJitter, } diff --git a/src/Polly.Core/Retry/RetryHelper.cs b/src/Polly.Core/Retry/RetryHelper.cs index e83d172ce8f..a221e929fe0 100644 --- a/src/Polly.Core/Retry/RetryHelper.cs +++ b/src/Polly.Core/Retry/RetryHelper.cs @@ -2,17 +2,30 @@ namespace Polly.Retry; internal static class RetryHelper { + private const double JitterFactor = 0.5; + private const double ExponentialFactor = 2.0; public static bool IsValidDelay(TimeSpan delay) => delay >= TimeSpan.Zero; - public static TimeSpan GetRetryDelay(RetryBackoffType type, int attempt, TimeSpan baseDelay, ref double state, Func randomizer) + public static TimeSpan GetRetryDelay(RetryBackoffType type, bool jitter, int attempt, TimeSpan baseDelay, ref double state, Func randomizer) { if (baseDelay == TimeSpan.Zero) { return baseDelay; } + if (jitter) + { + return type switch + { + RetryBackoffType.Constant => ApplyJitter(baseDelay, randomizer), + RetryBackoffType.Linear => ApplyJitter(TimeSpan.FromMilliseconds((attempt + 1) * baseDelay.TotalMilliseconds), randomizer), + RetryBackoffType.Exponential => DecorrelatedJitterBackoffV2(attempt, baseDelay, ref state, randomizer), + _ => throw new ArgumentOutOfRangeException(nameof(type), type, "The retry backoff type is not supported.") + }; + } + return type switch { RetryBackoffType.Constant => baseDelay, @@ -23,7 +36,6 @@ public static TimeSpan GetRetryDelay(RetryBackoffType type, int attempt, TimeSpa RetryBackoffType.Linear => (attempt + 1) * baseDelay, RetryBackoffType.Exponential => Math.Pow(ExponentialFactor, attempt) * baseDelay, #endif - RetryBackoffType.ExponentialWithJitter => DecorrelatedJitterBackoffV2(attempt, baseDelay, ref state, randomizer), _ => throw new ArgumentOutOfRangeException(nameof(type), type, "The retry backoff type is not supported.") }; } @@ -72,4 +84,7 @@ private static TimeSpan DecorrelatedJitterBackoffV2(int attempt, TimeSpan baseDe return TimeSpan.FromTicks((long)Math.Min(formulaIntrinsicValue * RpScalingFactor * targetTicksFirstDelay, maxTimeSpanDouble)); } + + private static TimeSpan ApplyJitter(TimeSpan delay, Func randomizer) + => TimeSpan.FromMilliseconds(delay.TotalMilliseconds + ((delay.TotalMilliseconds * JitterFactor) * randomizer())); } diff --git a/src/Polly.Core/Retry/RetryResilienceStrategy.cs b/src/Polly.Core/Retry/RetryResilienceStrategy.cs index f4224aa1625..802594c55c2 100644 --- a/src/Polly.Core/Retry/RetryResilienceStrategy.cs +++ b/src/Polly.Core/Retry/RetryResilienceStrategy.cs @@ -23,6 +23,7 @@ public RetryResilienceStrategy( RetryCount = options.RetryCount; OnRetry = options.OnRetry; DelayGenerator = options.RetryDelayGenerator; + UseJitter = options.UseJitter; _timeProvider = timeProvider; _telemetry = telemetry; @@ -39,6 +40,8 @@ public RetryResilienceStrategy( public Func, ValueTask>? DelayGenerator { get; } + public bool UseJitter { get; } + public Func, ValueTask>? OnRetry { get; } protected override async ValueTask> ExecuteCallbackAsync(Func>> callback, ResilienceContext context, TState state) @@ -62,7 +65,7 @@ protected override async ValueTask> ExecuteCallbackAsync(Func return outcome; } - var delay = RetryHelper.GetRetryDelay(BackoffType, attempt, BaseDelay, ref retryState, _randomizer); + var delay = RetryHelper.GetRetryDelay(BackoffType, UseJitter, attempt, BaseDelay, ref retryState, _randomizer); if (DelayGenerator is not null) { var delayArgs = new OutcomeArguments(context, outcome, new RetryDelayArguments(attempt, delay)); diff --git a/src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs b/src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs index d4ed56d267c..ebf3267d18e 100644 --- a/src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs +++ b/src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs @@ -34,6 +34,18 @@ public class RetryStrategyOptions : ResilienceStrategyOptions /// public RetryBackoffType BackoffType { get; set; } = RetryConstants.DefaultBackoffType; + /// + /// Gets or sets a value indicating whether the jitter should be used when calculating the backoff delay between retries. + /// + /// + /// See for more details + /// on how jitter can improve the resilience when the retries are correlated. + /// + /// + /// The default value is . + /// + public bool UseJitter { get; set; } + #pragma warning disable IL2026 // Addressed with DynamicDependency on ValidationHelper.Validate method /// /// Gets or sets the base delay between retries. @@ -45,9 +57,6 @@ public class RetryStrategyOptions : ResilienceStrategyOptions /// : Represents the median delay to target before the first retry. /// /// - /// : Represents the median delay to target before the first retry. - /// - /// /// : Represents the initial delay, the following delays increasing linearly with this value. /// /// diff --git a/test/Polly.Core.Tests/Retry/RetryHelperTests.cs b/test/Polly.Core.Tests/Retry/RetryHelperTests.cs index 4cef278b6e3..641a245e63d 100644 --- a/test/Polly.Core.Tests/Retry/RetryHelperTests.cs +++ b/test/Polly.Core.Tests/Retry/RetryHelperTests.cs @@ -6,7 +6,7 @@ namespace Polly.Core.Tests.Retry; public class RetryHelperTests { - private readonly Func _randomizer = new RandomUtil(0).NextDouble; + private Func _randomizer = new RandomUtil(0).NextDouble; [Fact] public void IsValidDelay_Ok() @@ -18,44 +18,67 @@ public void IsValidDelay_Ok() RetryHelper.IsValidDelay(TimeSpan.FromMilliseconds(-1)).Should().BeFalse(); } - [Fact] - public void UnsupportedRetryBackoffType_Throws() + [InlineData(true)] + [InlineData(false)] + [Theory] + public void UnsupportedRetryBackoffType_Throws(bool jitter) { RetryBackoffType type = (RetryBackoffType)99; Assert.Throws(() => { double state = 0; - return RetryHelper.GetRetryDelay(type, 0, TimeSpan.FromSeconds(1), ref state, _randomizer); + return RetryHelper.GetRetryDelay(type, jitter, 0, TimeSpan.FromSeconds(1), ref state, _randomizer); }); } - [Fact] - public void Constant_Ok() + [InlineData(true)] + [InlineData(false)] + [Theory] + public void Constant_Ok(bool jitter) { double state = 0; + if (jitter) + { + _randomizer = () => 0.5; + } + + RetryHelper.GetRetryDelay(RetryBackoffType.Constant, jitter, 0, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + RetryHelper.GetRetryDelay(RetryBackoffType.Constant, jitter, 1, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + RetryHelper.GetRetryDelay(RetryBackoffType.Constant, jitter, 2, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); - RetryHelper.GetRetryDelay(RetryBackoffType.Constant, 0, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); - RetryHelper.GetRetryDelay(RetryBackoffType.Constant, 1, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); - RetryHelper.GetRetryDelay(RetryBackoffType.Constant, 2, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + var expected = !jitter ? TimeSpan.FromSeconds(1) : TimeSpan.FromSeconds(1.25); - RetryHelper.GetRetryDelay(RetryBackoffType.Constant, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1)); - RetryHelper.GetRetryDelay(RetryBackoffType.Constant, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1)); - RetryHelper.GetRetryDelay(RetryBackoffType.Constant, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1)); + RetryHelper.GetRetryDelay(RetryBackoffType.Constant, jitter, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(expected); + RetryHelper.GetRetryDelay(RetryBackoffType.Constant, jitter, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(expected); + RetryHelper.GetRetryDelay(RetryBackoffType.Constant, jitter, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(expected); } - [Fact] - public void Linear_Ok() + [InlineData(true)] + [InlineData(false)] + [Theory] + public void Linear_Ok(bool jitter) { double state = 0; - RetryHelper.GetRetryDelay(RetryBackoffType.Linear, 0, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); - RetryHelper.GetRetryDelay(RetryBackoffType.Linear, 1, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); - RetryHelper.GetRetryDelay(RetryBackoffType.Linear, 2, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + RetryHelper.GetRetryDelay(RetryBackoffType.Linear, jitter, 0, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + RetryHelper.GetRetryDelay(RetryBackoffType.Linear, jitter, 1, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + RetryHelper.GetRetryDelay(RetryBackoffType.Linear, jitter, 2, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + + if (jitter) + { + _randomizer = () => 0.5; - RetryHelper.GetRetryDelay(RetryBackoffType.Linear, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1)); - RetryHelper.GetRetryDelay(RetryBackoffType.Linear, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2)); - RetryHelper.GetRetryDelay(RetryBackoffType.Linear, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(3)); + RetryHelper.GetRetryDelay(RetryBackoffType.Linear, jitter, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1.25)); + RetryHelper.GetRetryDelay(RetryBackoffType.Linear, jitter, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2.5)); + RetryHelper.GetRetryDelay(RetryBackoffType.Linear, jitter, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(3.75)); + } + else + { + RetryHelper.GetRetryDelay(RetryBackoffType.Linear, jitter, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1)); + RetryHelper.GetRetryDelay(RetryBackoffType.Linear, jitter, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2)); + RetryHelper.GetRetryDelay(RetryBackoffType.Linear, jitter, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(3)); + } } [Fact] @@ -63,13 +86,13 @@ public void Exponential_Ok() { double state = 0; - RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, 0, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); - RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, 1, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); - RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, 2, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, false, 0, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, false, 1, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); + RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, false, 2, TimeSpan.Zero, ref state, _randomizer).Should().Be(TimeSpan.Zero); - RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1)); - RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2)); - RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(4)); + RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, false, 0, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(1)); + RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, false, 1, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(2)); + RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, false, 2, TimeSpan.FromSeconds(1), ref state, _randomizer).Should().Be(TimeSpan.FromSeconds(4)); } [InlineData(1)] @@ -113,7 +136,7 @@ private static IReadOnlyList GetExponentialWithJitterBackoff(bool cont for (int i = 0; i < retryCount; i++) { - result.Add(RetryHelper.GetRetryDelay(RetryBackoffType.ExponentialWithJitter, i, baseDelay, ref state, random)); + result.Add(RetryHelper.GetRetryDelay(RetryBackoffType.Exponential, true, i, baseDelay, ref state, random)); } return result; diff --git a/test/Polly.Core.Tests/Retry/RetryResilienceStrategyBuilderExtensionsTests.cs b/test/Polly.Core.Tests/Retry/RetryResilienceStrategyBuilderExtensionsTests.cs index 172081d597f..c652fb260d3 100644 --- a/test/Polly.Core.Tests/Retry/RetryResilienceStrategyBuilderExtensionsTests.cs +++ b/test/Polly.Core.Tests/Retry/RetryResilienceStrategyBuilderExtensionsTests.cs @@ -107,7 +107,7 @@ public void AddRetry_InvalidOptions_Throws() [Fact] public void GetAggregatedDelay_ShouldReturnTheSameValue() { - var options = new RetryStrategyOptions { BackoffType = RetryBackoffType.ExponentialWithJitter }; + var options = new RetryStrategyOptions { BackoffType = RetryBackoffType.Exponential, UseJitter = true }; var delay = GetAggregatedDelay(options); GetAggregatedDelay(options).Should().Be(delay); diff --git a/test/Polly.Extensions.Tests/Issues/IssuesTests.StrategiesPerEndpoint_1365.cs b/test/Polly.Extensions.Tests/Issues/IssuesTests.StrategiesPerEndpoint_1365.cs index 13e4445e70d..66e3045baea 100644 --- a/test/Polly.Extensions.Tests/Issues/IssuesTests.StrategiesPerEndpoint_1365.cs +++ b/test/Polly.Extensions.Tests/Issues/IssuesTests.StrategiesPerEndpoint_1365.cs @@ -49,7 +49,8 @@ public void StrategiesPerEndpoint_1365() { builder.AddRetry(new() { - BackoffType = RetryBackoffType.ExponentialWithJitter, + BackoffType = RetryBackoffType.Exponential, + UseJitter = true, RetryCount = endpointOptions.Retries, StrategyName = $"{context.StrategyKey.EndpointName}-Retry", }); From 044d6a5c14d7ef5d75e5ee7ac971d9311ba5415e Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Fri, 28 Jul 2023 07:50:33 +0200 Subject: [PATCH 2/2] PR comments --- src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs b/src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs index ebf3267d18e..d984bc014f7 100644 --- a/src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs +++ b/src/Polly.Core/Retry/RetryStrategyOptions.TResult.cs @@ -35,7 +35,7 @@ public class RetryStrategyOptions : ResilienceStrategyOptions public RetryBackoffType BackoffType { get; set; } = RetryConstants.DefaultBackoffType; /// - /// Gets or sets a value indicating whether the jitter should be used when calculating the backoff delay between retries. + /// Gets or sets a value indicating whether jitter should be used when calculating the backoff delay between retries. /// /// /// See for more details