Merge pull request #515 from microsoft/release/update/250611105136

Added UseExponentialRetryDelayForConcurrencyThrottle configuration option

Author: tehcrashxor

src/GeneralTools/DataverseClient/Client/ConnectionService.cs

@@ -2475,14 +2475,19 @@ private bool ShouldRetryWebAPI(Exception ex, int retryCount, int maxRetryCount,
                         errorCode == ((int)ErrorCodes.ThrottlingTimeExceededError).ToString() ||
                         errorCode == ((int)ErrorCodes.ThrottlingConcurrencyLimitExceededError).ToString())
                     {
-                        if (httpOperationException.Response.Headers.TryGetValue("Retry-After", out var retryAfter) && double.TryParse(retryAfter.FirstOrDefault(), out var retrySeconds))
-                        { 
+                        if (_configuration.Value.UseExponentialRetryDelayForConcurrencyThrottle && errorCode == ((int)ErrorCodes.ThrottlingConcurrencyLimitExceededError).ToString())
+                        {
+                            // Use exponential delay for concurrency throttling if UseExponentialRetryDelayForConcurrencyThrottle is set to true
+                            _retryPauseTimeRunning = retryPauseTime.Add(TimeSpan.FromSeconds(Math.Pow(2, retryCount)));
+                        }
+                        else if (httpOperationException.Response.Headers.TryGetValue("Retry-After", out var retryAfter) && double.TryParse(retryAfter.FirstOrDefault(), out var retrySeconds))
+                        {
                             // Note: Retry-After header is in seconds.
-                             _retryPauseTimeRunning = TimeSpan.FromSeconds(retrySeconds);
+                            _retryPauseTimeRunning = TimeSpan.FromSeconds(retrySeconds);
                         }
                         else
                         { 
-                            _retryPauseTimeRunning = retryPauseTime.Add(TimeSpan.FromSeconds(Math.Pow(2, retryCount))); ; // default timespan with back off is response does not contain the tag..
+                            _retryPauseTimeRunning = retryPauseTime.Add(TimeSpan.FromSeconds(Math.Pow(2, retryCount))); // default timespan with back off is response does not contain the tag..
                         }
                         isThrottlingRetry = true;
                         return true;
@@ -2595,6 +2600,7 @@ private bool ShouldRetryWebAPI(Exception ex, int retryCount, int maxRetryCount,
                             }
                         }
                         catch { }
+                        // CodeQL [SM03781] By Design - this is a client library and users need to be able to target any relevant URI
                         _httpResponse = await providedHttpClient.SendAsync(_httpRequest, cancellationToken).ConfigureAwait(false);
                         logDt.Stop();
                     }
@@ -2613,6 +2619,7 @@ private bool ShouldRetryWebAPI(Exception ex, int retryCount, int maxRetryCount,
                                 }
                             }
                             catch { }
+                            // CodeQL [SM03781] By Design - this is a client library and users need to be able to target any relevant URI
                             _httpResponse = await httpCli.SendAsync(_httpRequest, cancellationToken).ConfigureAwait(false);
                             logDt.Stop();
                         }

src/GeneralTools/DataverseClient/Client/Model/ConfigurationOptions.cs

@@ -33,6 +33,7 @@ public void UpdateOptions(ConfigurationOptions options)
                 RetryPauseTime = options.RetryPauseTime;
                 UseWebApi = options.UseWebApi;
                 UseWebApiLoginFlow = options.UseWebApiLoginFlow;
+                UseExponentialRetryDelayForConcurrencyThrottle = options.UseExponentialRetryDelayForConcurrencyThrottle;
             }
         }
 
@@ -71,6 +72,17 @@ public bool UseWebApi
             set => _useWebApi = value;
         }
 
+        private bool _useExponentialRetryDelayForConcurrencyThrottle = Utils.AppSettingsHelper.GetAppSetting<bool>("UseExponentialRetryDelayForConcurrencyThrottle", false);
+
+        /// <summary>
+        /// Use exponential retry delay for concurrency throttling instead of server specified Retry-After header
+        /// </summary>
+        public bool UseExponentialRetryDelayForConcurrencyThrottle
+        {
+            get => _useExponentialRetryDelayForConcurrencyThrottle;
+            set => _useExponentialRetryDelayForConcurrencyThrottle = value;
+        }
+
         private bool _useWebApiLoginFlow = Utils.AppSettingsHelper.GetAppSetting<bool>("UseWebApiLoginFlow", true);
         /// <summary>
         /// Use Web API instead of org service for logging into and getting boot up data.

src/GeneralTools/DataverseClient/Client/ServiceClient.cs

@@ -207,6 +207,15 @@ public TimeSpan RetryPauseTime
             set { _configuration.Value.RetryPauseTime = value; }
         }
 
+        /// <summary>
+        /// Use exponential retry delay for concurrency throttling instead of server specified Retry-After header where possible - Defaults to False.
+        /// </summary>
+        public bool UseExponentialRetryDelayForConcurrencyThrottle
+        {
+            get { return _configuration.Value.UseExponentialRetryDelayForConcurrencyThrottle; }
+            set { _configuration.Value.UseExponentialRetryDelayForConcurrencyThrottle = value; }
+        }
+
         /// <summary>
         /// if true the service is ready to accept requests.
         /// </summary>
@@ -1417,6 +1426,7 @@ public ServiceClient Clone(System.Reflection.Assembly strongTypeAsm, ILogger log
                             SvcClient.CallerId = CallerId;
                             SvcClient.MaxRetryCount = _configuration.Value.MaxRetryCount;
                             SvcClient.RetryPauseTime = _configuration.Value.RetryPauseTime;
+                            SvcClient.UseExponentialRetryDelayForConcurrencyThrottle = _configuration.Value.UseExponentialRetryDelayForConcurrencyThrottle;
                             SvcClient.GetAccessToken = GetAccessToken;
                             SvcClient.GetCustomHeaders = GetCustomHeaders;
                             return SvcClient;
@@ -2055,9 +2065,14 @@ private bool ShouldRetry(OrganizationRequest req, Exception ex, int retryCount,
                     OrgEx.Detail.ErrorCode == ErrorCodes.ThrottlingTimeExceededError ||
                     OrgEx.Detail.ErrorCode == ErrorCodes.ThrottlingConcurrencyLimitExceededError)
                 {
-                    // Use Retry-After delay when specified
-                    if (OrgEx.Detail.ErrorDetails.TryGetValue("Retry-After", out var retryAfter) && retryAfter is TimeSpan retryAsTimeSpan)
+                    if (_configuration.Value.UseExponentialRetryDelayForConcurrencyThrottle && OrgEx.Detail.ErrorCode == ErrorCodes.ThrottlingConcurrencyLimitExceededError)
+                    {
+                        // Use exponential delay for concurrency throttling if UseExponentialRetryDelayForConcurrencyThrottle is set to true
+                        _retryPauseTimeRunning = _configuration.Value.RetryPauseTime.Add(TimeSpan.FromSeconds(Math.Pow(2, retryCount)));
+                    }
+                    else if (OrgEx.Detail.ErrorDetails.TryGetValue("Retry-After", out var retryAfter) && retryAfter is TimeSpan retryAsTimeSpan)
                     {
+                        // Use Retry-After delay when specified
                         _retryPauseTimeRunning = retryAsTimeSpan;
                     }
                     else

src/GeneralTools/DataverseClient/Extensions/Microsoft.PowerPlatform.Dataverse.Client.AzAuth/AzAuth.cs

@@ -190,6 +190,7 @@ private async Task<Uri> InitializeCredentials(Uri instanceUrl)
                 _logger.LogDebug("Initialize Creds - found resource with name " + (string.IsNullOrEmpty(authDetails.Resource.ToString()) ? "<Not Provided>" : authDetails.Resource.ToString()));
                 _logger.LogDebug("Initialize Creds - found tenantId " + (string.IsNullOrEmpty(_credentialOptions.TenantId) ? "<Not Provided>" : _credentialOptions.TenantId));
             }
+            // CodeQL [SM05137] Not applicable - this is a Public client SDK
             _defaultAzureCredential = new DefaultAzureCredential(_credentialOptions);
 
             _logger.LogDebug("Credentials initialized in {0}ms", sw.ElapsedMilliseconds);

src/GeneralTools/DataverseClient/UnitTests/CdsClient_Core_Tests/ServiceClientTests.cs

@@ -884,6 +884,18 @@ public async Task RetryOperationShouldNotThrowWhenAlreadyCanceledTest()
             testwatch.Elapsed.Should().BeLessThan(delay, "Task should return before its delay timer can complete due to cancellation");
         }
 
+        [Fact]
+        public void TestOptionUseExponentialRetryDelayForConcurrencyThrottle()
+        {
+            testSupport.SetupMockAndSupport(out var orgSvc, out var fakHttpMethodHander, out var cli);
+            cli.UseExponentialRetryDelayForConcurrencyThrottle = true;
+
+            var rsp = (WhoAmIResponse)cli.ExecuteOrganizationRequest(new WhoAmIRequest());
+
+            // Validate that the behavior remains unchanged when the option UseExponentialRetryDelayForConcurrencyThrottle is set to true.
+            Assert.Equal(rsp.UserId, testSupport._UserId);
+        }
+
         #region LiveConnectedTests
 
         [SkippableConnectionTest]

src/nuspecs/Microsoft.PowerPlatform.Dataverse.Client.ReleaseNotes.txt

@@ -7,6 +7,9 @@ Notice:
     Note: Only AD on FullFramework, OAuth, Certificate, ClientSecret Authentication types are supported at this time.
 
 ++CURRENTRELEASEID++
+Add a new configuration option UseExponentialRetryDelayForConcurrencyThrottle to use exponential retry delay for concurrency throttling, instead of the server-specified Retry-After header where applicable. Default is False.
+
+1.2.7
 Fix for CancellationToken not canceling retries during delays Git: https://github.com/microsoft/PowerPlatform-DataverseServiceClient/issues/508
 
 1.2.5