internal/functions/core/Invoke-EntraRetry.ps1
|
function Invoke-EntraRetry { <# .SYNOPSIS Runs one Graph call and repeats it as the retry policy of the module allows. .DESCRIPTION The single retry layer for one call. The call is repeated after a failure that repeating can help: - an answer whose status Test-EntraRetryStatus calls transient (408, 429, 5xx except 501 and 505, and a 400 whose Graph error code - read from the body in ErrorDetails - is Directory_ConcurrencyViolation); - no answer at all (a network failure, a timeout). A call that is not idempotent (-NotIdempotent: a POST) is repeated only on 429 and 503 - the service said it did not carry it out; after a 500 or a lost answer it may have been done. The wait before the next attempt is the Retry-After of the answer (seconds or an HTTP date) when it has one, at most 5 minutes; otherwise 2^attempt seconds with a jitter of 0.8 to 1.2, also at most 5 minutes, so many callers throttled at once do not come back at the same moment. The number of repetitions is Settings.Request.RetryCount (or -RetryCount). The last failure is thrown as it is. Invoke-EntraRequest sends every page through this; nothing above it retries (the cmdlets call Invoke-PSFProtectedCommand without -RetryCount), so a request is sent at most RetryCount + 1 times. Every variable of this function starts with entraRetry: the script block runs in a scope below this one, and a name it reads from its caller (a parameter $Header, say) must not be shadowed by one of these. .PARAMETER ScriptBlock The call. It throws its failure (Invoke-RestMethod -ErrorAction Stop or an exception whose Response carries the HTTP answer). .PARAMETER NotIdempotent The call may change something: repeat it only when the service said it did not carry it out (429, 503). .PARAMETER RetryCount How often a failed call is repeated; default Settings.Request.RetryCount. .PARAMETER Report The run report (see Write-EntraBatchSummary): every repetition is counted under Resent with its pause, and a call that got through after one under Resent.Recovered. .EXAMPLE PS C:\> Invoke-EntraRetry -ScriptBlock { Invoke-RestMethod -Uri $uri -Headers $header -ErrorAction Stop } Reads the URL, repeating the read on throttling, server errors and lost answers. #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [scriptblock] $ScriptBlock, [switch] $NotIdempotent, [ValidateRange(0, 20)] [int] $RetryCount, [hashtable] $Report ) # Names of their own: the script block runs in a scope below this one and sees the variables of its caller. [int] $entraRetryCount = if ($PSBoundParameters.ContainsKey('RetryCount')) { $RetryCount } else { Get-PSFConfigValue -FullName 'PSMicrosoftEntraID.Settings.Request.RetryCount' -Fallback 5 } [int] $entraRetryAttempt = 0 while ($true) { try { $entraRetryResult = & $ScriptBlock if ($entraRetryAttempt -gt 0 -and $null -ne $Report) { $Report['Resent.Recovered'] = [int]$Report['Resent.Recovered'] + 1 } break } catch { $entraRetryFailure = $_ # The answer, if any: the first exception of the chain that carries an HTTP response. [int] $entraRetryStatus = -1 [double] $entraRetryAfter = 0 [bool] $entraRetryNoAnswer = $false $entraRetryException = $entraRetryFailure.Exception while ($entraRetryException) { $entraRetryResponse = $null if ($entraRetryException.PSObject.Properties['Response']) { $entraRetryResponse = $entraRetryException.Response } if ($entraRetryResponse -and $entraRetryResponse.PSObject.Properties['StatusCode']) { $entraRetryStatus = [int]$entraRetryResponse.StatusCode if ($entraRetryResponse.PSObject.Properties['Headers'] -and $entraRetryResponse.Headers -and $entraRetryResponse.Headers.PSObject.Properties['RetryAfter']) { $entraRetryHeader = $entraRetryResponse.Headers.RetryAfter if ($entraRetryHeader -and $entraRetryHeader.Delta) { $entraRetryAfter = $entraRetryHeader.Delta.TotalSeconds } elseif ($entraRetryHeader -and $entraRetryHeader.Date) { $entraRetryAfter = ($entraRetryHeader.Date - [DateTimeOffset]::UtcNow).TotalSeconds } } break } if ($entraRetryException -is [System.Net.Http.HttpRequestException] -and $entraRetryException.StatusCode) { $entraRetryStatus = [int]$entraRetryException.StatusCode break } if ($entraRetryException -is [System.Net.Http.HttpRequestException] -or $entraRetryException -is [System.TimeoutException] -or $entraRetryException -is [System.Threading.Tasks.TaskCanceledException] -or $entraRetryException -is [System.IO.IOException]) { $entraRetryNoAnswer = $true break } $entraRetryException = $entraRetryException.InnerException } # The Graph error codes of the answer's body (Invoke-RestMethod puts it into ErrorDetails): a 400 # Directory_ConcurrencyViolation is transient. Only the codes are read, never the message text. [string[]] $entraRetryCodes = @() if ($entraRetryStatus -eq 400 -and $entraRetryFailure.ErrorDetails -and $entraRetryFailure.ErrorDetails.Message) { try { $entraRetryBody = $entraRetryFailure.ErrorDetails.Message | ConvertFrom-Json -ErrorAction Stop $entraRetryCodes = @($entraRetryBody.error.code, $entraRetryBody.error.innerError.code) | Where-Object { $_ -is [string] } } catch { $entraRetryCodes = @() } } [bool] $entraRetry = if ($entraRetryStatus -ge 0) { Test-EntraRetryStatus -Status $entraRetryStatus -ErrorCode $entraRetryCodes -NotIdempotent:$NotIdempotent } elseif ($entraRetryNoAnswer) { -not $NotIdempotent } else { $false } if (-not $entraRetry -or $entraRetryAttempt -ge $entraRetryCount) { throw } $entraRetryAttempt++ # What the service asked for, at most 5 minutes; without it 2^attempt seconds with jitter, at most 5 minutes. [double] $entraRetryWait = [PSMicrosoftEntraID.Batch.RetryAfter]::Limit($entraRetryAfter) if ($entraRetryWait -le 0) { $entraRetryWait = [Math]::Min([double][PSMicrosoftEntraID.Batch.RetryAfter]::MaximumSeconds, [Math]::Pow(2, $entraRetryAttempt) * (0.8 + 0.4 * [System.Random]::Shared.NextDouble())) } if ($null -ne $Report) { $Report['Resent'] = [int]$Report['Resent'] + 1 $Report['Resent.Longest'] = [Math]::Max([int]$Report['Resent.Longest'], [int][Math]::Ceiling($entraRetryWait)) if ($entraRetryStatus -eq 429) { $Report['Throttled'] = [int]$Report['Throttled'] + 1 } } [string] $entraRetryReason = if ($entraRetryStatus -ge 0) { [string]$entraRetryStatus } else { 'no answer' } Write-PSFMessage -Level Verbose -String 'Request.Retry' -StringValues $entraRetryReason, ([Math]::Round($entraRetryWait, 1)), $entraRetryAttempt, $entraRetryCount Start-Sleep -Milliseconds ([int][Math]::Ceiling($entraRetryWait * 1000)) } } $entraRetryResult } |