functions/Invoke-RestMethodWithRateLimit.ps1
|
# <copyright file="Invoke-RestMethodWithRateLimit.ps1" company="Endjin Limited"> # Copyright (c) Endjin Limited. All rights reserved. # </copyright> function Invoke-RestMethodWithRateLimit { <# .SYNOPSIS Invokes REST API calls with automatic rate limit handling and exponential backoff retry logic. .DESCRIPTION This wrapper function handles Fabric API and Microsoft Graph API rate limiting by: - Detecting HTTP 429 responses - Extracting Retry-After headers - Implementing exponential backoff with jitter - Providing configurable retry attempts - Maintaining detailed logging .PARAMETER Splat Hashtable containing all parameters for Invoke-RestMethod (Uri, Method, Headers, Body, etc.) .PARAMETER MaxRetries Maximum number of retry attempts (default: 3). .PARAMETER BaseDelaySeconds Base delay in seconds for exponential backoff (default: 1). .PARAMETER MaxDelaySeconds Maximum delay in seconds between retries (default: 60). .PARAMETER RetryBackOffExponentialFactor The factor used when calculating how quickly the exponential backoff should increase. Defaults to 1.5. .OUTPUTS System.Object .EXAMPLE ```powershell $splat = @{ Uri = "https://api.fabric.microsoft.com/v1/connections" Method = "GET" Headers = @{ Authorization = "Bearer $token" } } $result = Invoke-RestMethodWithRateLimit -Splat $splat ``` .EXAMPLE ```powershell $result = Invoke-RestMethodWithRateLimit -Splat $splat -MaxRetries 5 -BaseDelaySeconds 2 -MaxDelaySeconds 120 ``` #> [CmdletBinding()] [OutputType([Object])] param ( [Parameter(Mandatory=$true)] [hashtable] $Splat, [Parameter()] [int] $MaxRetries = 3, [Parameter()] [double] $BaseDelaySeconds = 1.0, [Parameter()] [int] $MaxDelaySeconds = 60, [Parameter()] [double] $RetryBackOffExponentialFactor = 1.5 ) $attempt = 0 $lastException = $null while ($attempt -le $MaxRetries) { try { Write-Verbose "API call attempt $($attempt + 1) to $($Splat.Uri)" $response = Invoke-RestMethod @Splat if ($attempt -gt 0) { Write-Information "API call succeeded after $($attempt + 1) attempts" } return $response } catch { $lastException = $_ $statusCode = $null $retryAfter = $null # Extract status code and retry-after header if available if ($_.Exception.Response) { $statusCode = $_.Exception.Response.StatusCode.value__ # Look for rate limiting headers using the extracted validation function if ($_.Exception.Response.Headers) { # Treat 'Retry-After' header as taking precedence as it is the only such mechanism in the current specification # Ref: https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.3 $retryAfterValue = Get-HttpHeaderValue -Headers $_.Exception.Response.Headers -HeaderName 'Retry-After' -ExpectedType ([int]) -DefaultValue $null if ($retryAfterValue) { $retryAfter = $retryAfterValue } else { # Whilst no longer part of a current specification, the 'X-RateLimit-*' headers are used in the real world (e.g. GitHub) $rateLimitRemaining = Get-HttpHeaderValue -Headers $_.Exception.Response.Headers -HeaderName 'X-RateLimit-Remaining' -ExpectedType ([int]) -DefaultValue $null $rateLimitResetFileTime = Get-HttpHeaderValue -Headers $_.Exception.Response.Headers -HeaderName 'X-RateLimit-Reset' -ExpectedType ([long]) -DefaultValue $null # When all the required headers are available, set the retry interval based on the 'X-RateLimit-Reset' header if ($null -ne $rateLimitRemaining -and $rateLimitRemaining -eq 0 -and $rateLimitResetFileTime) { try { $rateLimitResetDateTime = [datetime]::FromFileTimeUtc($rateLimitResetFileTime) $retryAfter = ($rateLimitResetDateTime - [datetime]::UtcNow).TotalSeconds # Only use the calculated retry interval if it's positive if ($retryAfter -gt 0) { Write-Verbose "Rate limit exceeded. Waiting for quota reset in $retryAfter seconds..." } else { Write-Verbose "X-RateLimit-Reset time is in the past. Falling back to exponential backoff." $retryAfter = $null } } catch { Write-Verbose "Could not convert X-RateLimit-Reset FileTime to DateTime: $rateLimitResetFileTime. Falling back to exponential backoff." } } } } } # Check if this is a retryable error (429 or 5xx) $isRetryable = ($statusCode -eq 429) -or ($statusCode -ge 500 -and $statusCode -lt 600) if (-not $isRetryable -or $attempt -eq $MaxRetries) { Write-Verbose "Non-retryable error or max retries reached. Status: $statusCode, Attempt: $($attempt + 1)" throw $lastException } $attempt++ # Calculate delay with exponential backoff and jitter if ($statusCode -eq 429 -and $retryAfter) { # Use server-provided retry-after value, or the 'MaxDelaySeconds' if it is shorter $delaySeconds = [Math]::Min($retryAfter, $MaxDelaySeconds) if ($retryAfter -gt $MaxDelaySeconds) { Write-Information "Rate limited (429). Server requested waiting $retryAfter seconds which exceeds MaxDelaySeconds. Waiting for $MaxDelaySeconds seconds." } else { Write-Information "Rate limited (429). Waiting $delaySeconds seconds as requested by server." } } else { # Use exponential backoff with jitter $exponentialDelay = [Math]::Pow($RetryBackOffExponentialFactor, $attempt - 1) * $BaseDelaySeconds $jitter = (Get-Random -Minimum 0.0 -Maximum 1.0) * 0.1 * $exponentialDelay $delaySeconds = [Math]::Min($exponentialDelay + $jitter, $MaxDelaySeconds) $errorType = if ($statusCode -eq 429) { "Rate limited" } else { "Server error ($statusCode)" } Write-Warning "$errorType. Retrying in $([Math]::Round($delaySeconds, 2)) seconds (attempt $attempt of $MaxRetries)" } Start-Sleep -Seconds $delaySeconds } } throw $lastException } |