private/Invoke-VSAHttp.ps1
|
# The module's single HTTP stack (F-67). # # Every request the module makes -- sequential or parallel -- is issued by System.Net.Http.HttpClient # and governed by the policy defined in this file. Before v1.5.1 there were two stacks: the # sequential path used Invoke-RestMethod while the parallel engine used HttpClient, each with its own # copy of the retry rules, envelope handling and error typing. Those copies had already drifted (the # parallel path ignored Retry-After, and surfaced raw exceptions instead of a typed VSAApiException), # so the duplication was not merely untidy -- it was a source of live defects. # # Layering: # Get-VSAHttpClient - the shared, cached HttpClient (edition-specific certificate strategy) # New-VSAHttpContent - request body -> HttpContent # Get-VSARetryAfterSeconds / Get-VSABackoffSeconds / $script:VSARetryStatuses - the retry policy # Resolve-VSAResponse - the API envelope rules # Invoke-VSAHttp - blocking send + retry (used by Get-RequestData) # Invoke-VSAParallelRequest reuses every piece above except Invoke-VSAHttp itself, substituting its # own asynchronous pump for the blocking send. That is the only intentional difference between the # two paths. # ONE definition of the transient-retry status set, shared by both dispatch modes. $script:VSARetryStatuses = @(429, 502, 503, 504) # HttpClient is designed to be long-lived and shared; constructing one per request exhausts sockets # (each disposed client leaves a connection in TIME_WAIT). Cache one per certificate strategy. $script:VSAHttpClients = @{} function Get-VSAHttpClient { <# .SYNOPSIS Returns the shared HttpClient for the requested certificate strategy, creating it on first use. .DESCRIPTION Certificate bypass is edition-specific, mirroring the strategy feature-detected once at import (F-27): - PowerShell 7 / .NET Core: a per-handler validation callback. ServicePointManager is ignored on Core, so the bypass must live on the handler and is scoped to this client. - Windows PowerShell 5.1 / .NET Framework: HttpClient is built on HttpWebRequest and so honours the process-wide ICertificatePolicy that VSAPushCertBypass installs. The handler callback is deliberately NOT used there: it is unavailable on older Framework versions, and the process-wide policy is the path the module already proves out on 5.1. The returned client MUST NOT be disposed by callers -- it is shared and cached. .PARAMETER IgnoreCertificateErrors Selects the certificate-bypassing client rather than the strict one. .OUTPUTS System.Net.Http.HttpClient #> [CmdletBinding()] [OutputType([System.Net.Http.HttpClient])] param( [Parameter(Mandatory = $false)] [bool] $IgnoreCertificateErrors = $false ) $key = if ($IgnoreCertificateErrors) { 'bypass' } else { 'strict' } if ($script:VSAHttpClients.ContainsKey($key)) { return $script:VSAHttpClients[$key] } # System.Net.Http is loaded at module import (VSAModule.psm1), NOT here. On .NET Framework it is # a separate assembly, and PowerShell resolves this function's type literals when it compiles the # body on first invocation -- an Add-Type at this point would already be too late to satisfy them. $handler = [System.Net.Http.HttpClientHandler]::new() if ($IgnoreCertificateErrors -and $script:VSASupportsSkipCertCheck) { $handler.ServerCertificateCustomValidationCallback = [System.Net.Http.HttpClientHandler]::DangerousAcceptAnyServerCertificateValidator } # Concurrency is governed by the parallel pump's own in-flight window; these caps only need to be # wide enough not to throttle it behind the platform default (2 per host on .NET Framework). try { $handler.MaxConnectionsPerServer = 64 } catch { Write-Debug "MaxConnectionsPerServer unavailable on this runtime: $($_.Exception.Message)" } if (-not $script:VSASupportsSkipCertCheck) { try { [System.Net.ServicePointManager]::DefaultConnectionLimit = [Math]::Max([System.Net.ServicePointManager]::DefaultConnectionLimit, 64) } catch { Write-Debug "Could not raise DefaultConnectionLimit: $($_.Exception.Message)" } } $client = [System.Net.Http.HttpClient]::new($handler) # Timeouts are enforced per request via a CancellationToken, because this client is shared and # HttpClient.Timeout is a client-wide property that cannot vary per call. $client.Timeout = [System.Threading.Timeout]::InfiniteTimeSpan $script:VSAHttpClients[$key] = $client return $client } function New-VSAHttpContent { <# .SYNOPSIS Converts a request body ([string] JSON or [byte[]] raw/multipart) into an HttpContent. .DESCRIPTION Always uses ByteArrayContent and sets Content-Type verbatim via TryAddWithoutValidation. StringContent is deliberately avoided: it appends its own charset parameter, which would corrupt a multipart Content-Type whose boundary= parameter must survive byte-for-byte (F-37). .PARAMETER Body [string] for JSON bodies, [byte[]] for raw/multipart bodies. .PARAMETER ContentType The Content-Type header to send verbatim. .OUTPUTS System.Net.Http.HttpContent, or $null when there is no body. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '')] [CmdletBinding()] param( [Parameter(Mandatory = $false)] [object] $Body, [Parameter(Mandatory = $false)] [string] $ContentType = 'application/json' ) if ($null -eq $Body) { return $null } [byte[]] $bytes = if ($Body -is [byte[]]) { $Body } else { [System.Text.Encoding]::UTF8.GetBytes([string]$Body) } $content = [System.Net.Http.ByteArrayContent]::new($bytes) $content.Headers.Remove('Content-Type') | Out-Null $content.Headers.TryAddWithoutValidation('Content-Type', $ContentType) | Out-Null return $content } function Get-VSARetryAfterSeconds { <# .SYNOPSIS Returns the Retry-After delay in seconds from an HttpResponseMessage, or $null if absent. .DESCRIPTION Handles both header forms: delta-seconds (Retry-After: 120) and an HTTP-date. A server that states its own back-off is always more accurate than client-side exponential guessing, so this value takes precedence in Get-VSABackoffSeconds. .PARAMETER Response The System.Net.Http.HttpResponseMessage to inspect. #> [CmdletBinding()] param( [Parameter(Mandatory = $false)] [object] $Response ) if ($null -eq $Response) { return $null } try { $ra = $Response.Headers.RetryAfter if ($null -eq $ra) { return $null } if ($null -ne $ra.Delta) { return [int][Math]::Ceiling($ra.Delta.TotalSeconds) } if ($null -ne $ra.Date) { $secs = [int][Math]::Ceiling(($ra.Date.UtcDateTime - [datetime]::UtcNow).TotalSeconds) if ($secs -gt 0) { return $secs } } } catch { Write-Debug "Retry-After parse skipped: $($_.Exception.Message)" } return $null } function Test-VSARetryable { <# .SYNOPSIS The single decision of "should this failed attempt be retried?", shared by both dispatch modes. .DESCRIPTION Two distinct cases, and the difference between them matters for correctness: 1. **A transient HTTP status (429/502/503/504).** The server answered and told us it did not process the request, so retrying is safe for ANY method. 2. **No HTTP response at all** (socket reset / timeout). Here we cannot know whether the server processed the request before the connection dropped, so retrying is only safe for an IDEMPOTENT method. Retrying a POST/PUT/DELETE could apply it twice. It is also the wrong call in practice: on hardened (post-2021) VSA builds a reset is precisely how a blocked write endpoint answers, so retrying merely spends the backoff before returning the same ConnectionReset. GET is the only method the module issues in bulk, so read fan-outs still survive a transient socket blip, while writes fail fast and exactly once. .PARAMETER StatusCode HTTP status, or 0 when no response arrived. .PARAMETER Method The HTTP method of the attempt. .PARAMETER NoResponse True when the attempt produced no HTTP response at all. #> [CmdletBinding()] [OutputType([bool])] param( [Parameter(Mandatory = $true)] [int] $StatusCode, [Parameter(Mandatory = $false)] [string] $Method = 'GET', [Parameter(Mandatory = $false)] [bool] $NoResponse = $false ) if ($StatusCode -in $script:VSARetryStatuses) { return $true } if ($NoResponse -and $Method -in @('GET', 'HEAD', 'OPTIONS')) { return $true } return $false } function Get-VSABackoffSeconds { <# .SYNOPSIS Returns how long to wait before retry attempt $Attempt, honouring a server Retry-After. .DESCRIPTION The single back-off rule for the whole module. A server-supplied Retry-After always wins; otherwise exponential (2^(Attempt-1)) capped at 30s. .PARAMETER Attempt The 1-based retry attempt about to be made. .PARAMETER RetryAfterSeconds The server's Retry-After value, if it supplied one. #> [CmdletBinding()] [OutputType([int])] param( [Parameter(Mandatory = $true)] [int] $Attempt, [Parameter(Mandatory = $false)] [AllowNull()] [object] $RetryAfterSeconds = $null ) if ($null -ne $RetryAfterSeconds -and [int]$RetryAfterSeconds -gt 0) { return [Math]::Min(30, [int]$RetryAfterSeconds) } return [int][Math]::Min(30, [Math]::Pow(2, [Math]::Max(0, $Attempt - 1))) } function Resolve-VSAResponse { <# .SYNOPSIS Applies the VSA API envelope rules to a deserialized 2xx response body. .DESCRIPTION The single definition of "what does a successful VSA response look like", shared by the sequential and parallel paths. Returns the value the caller should receive, or throws a typed VSAApiException for an application-level error carried inside an HTTP 200 envelope. Rules, in order: 1. Empty body (HTTP 204, returned by DELETE and some PUT endpoints) -> $null, a single empty-success sentinel for callers (F-21). 2. ResponseCode 4xx/5xx inside a 200 -> application error, throw typed. 3. ResponseCode 0/20x, or Status 'OK' -> success, return the envelope. 4. Neither ResponseCode nor Status present -> a raw, non-enveloped payload. Cloud Backup (kcb/*) returns a bare JSON map with none of the envelope fields; that is a successful result, not a broken envelope, so return it as-is (F-63). 5. Anything else -> an envelope we do not recognise; throw rather than return it silently. .PARAMETER Response The deserialized response body ($null for an empty body). .PARAMETER Method HTTP method, for error context. .PARAMETER Uri Request URI, for error context. #> [CmdletBinding()] param( [Parameter(Mandatory = $false)] [AllowNull()] [object] $Response, [Parameter(Mandatory = $false)] [string] $Method, [Parameter(Mandatory = $false)] [string] $Uri ) if ($null -eq $Response -or ($Response -is [string] -and [string]::IsNullOrWhiteSpace($Response))) { return $null } if ($Response.ResponseCode -match "(^40\d|^50\d)") { $code = 0; [void][int]::TryParse([string]$Response.ResponseCode, [ref]$code) throw (New-VSAApiError -Message "API Error (Code: $($Response.ResponseCode)) for $Method $Uri. Error: '$($Response.Error)'. Result: '$($Response.Result)'." ` -StatusCode $code -Method $Method -Uri $Uri -VSAError ([string]$Response.Error)) } if (($Response.ResponseCode -match "(^0$)|(^20\d+$)") -or ('OK' -eq $Response.Status)) { return $Response } if (($null -eq $Response.PSObject.Properties['ResponseCode']) -and ($null -eq $Response.PSObject.Properties['Status'])) { return $Response } throw (New-VSAApiError -Message "Unexpected API response for $Method $Uri. Response Code: '$($Response.ResponseCode)'. Error: '$($Response.Error)'." ` -StatusCode 0 -Method $Method -Uri $Uri -VSAError ([string]$Response.Error)) } function ConvertFrom-VSAErrorBody { <# .SYNOPSIS Extracts the VSA-specific "Error" field from a raw error-response body, or $null. .PARAMETER Body The raw response body text. #> [CmdletBinding()] param( [Parameter(Mandatory = $false)] [string] $Body ) if ([string]::IsNullOrWhiteSpace($Body)) { return $null } try { $parsed = $Body | ConvertFrom-Json -ErrorAction Stop if ($parsed.Error) { return [string]$parsed.Error } } catch { Write-Debug "Error body is not JSON: $($_.Exception.Message)" } return $null } function New-VSATransportError { <# .SYNOPSIS Builds the typed error for a terminal HTTP failure, shared by both dispatch modes. .DESCRIPTION Centralises the two terminal shapes so the parallel path can no longer surface a raw exception where the sequential path surfaces a typed VSAApiException: - StatusCode 0 (no HTTP response): the socket was reset or the endpoint is unreachable. On hardened (post-2021) VSA builds, user-mutation endpoints reset the connection here rather than returning 403/404, so ConnectionReset is set for callers to branch on. - Any other status: a real HTTP error, with the VSA error text if the body carried one. .PARAMETER StatusCode HTTP status, or 0 when no response arrived. .PARAMETER Method HTTP method. .PARAMETER Uri Request URI. .PARAMETER Body Raw response body, if any. .PARAMETER InnerException The underlying transport exception, if any. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '')] [CmdletBinding()] param( [Parameter(Mandatory = $true)] [int] $StatusCode, [Parameter(Mandatory = $false)] [string] $Method, [Parameter(Mandatory = $false)] [string] $Uri, [Parameter(Mandatory = $false)] [string] $Body, [Parameter(Mandatory = $false)] [Exception] $InnerException ) $vsaError = ConvertFrom-VSAErrorBody -Body $Body if ($StatusCode -eq 0) { $detail = @( "No HTTP response for $Method $Uri." "The connection was reset or the endpoint is unreachable -- it may be blocked or" "restricted on this VSA build (common for user-mutation endpoints on hardened," "post-2021 instances)." if ($InnerException) { "Underlying error: $($InnerException.Message)" } if ($Body) { "Response Body: $Body" } ) -join "`n" return (New-VSAApiError -Message $detail -StatusCode 0 -Method $Method -Uri $Uri ` -VSAError $vsaError -InnerException $InnerException -ConnectionReset) } $statusName = Get-VSAHttpStatusName -StatusCode $StatusCode $detail = @( "Failed to call REST API endpoint." "Method: $Method" "URI: $Uri" "HTTP Status: $StatusCode" "Error: Response status code does not indicate success: $StatusCode ($statusName)." if ($vsaError) { "VSA Error: $vsaError" } elseif ($Body) { "Response Body: $Body" } ) -join "`n" return (New-VSAApiError -Message $detail -StatusCode $StatusCode -Method $Method -Uri $Uri ` -VSAError $vsaError -InnerException $InnerException) } function Invoke-VSAHttp { <# .SYNOPSIS Issues a single HTTP request via the shared HttpClient, with the module's retry policy. .DESCRIPTION The blocking half of the module's single HTTP stack. Sends one request, retries transient failures per the shared policy, and returns the raw outcome. Envelope interpretation is NOT done here -- that is Resolve-VSAResponse's job -- so this function stays a pure transport. Unlike Invoke-RestMethod, HttpClient does not throw on 4xx/5xx, so the error body is simply read rather than reconstructed from an ErrorRecord. .PARAMETER Uri Absolute request URI. .PARAMETER Method HTTP method. .PARAMETER AuthString Full Authorization header value. .PARAMETER Body [string] JSON body, or [byte[]] raw/multipart body. .PARAMETER ContentType Content-Type for the body. .PARAMETER TimeoutSec Per-request timeout. .PARAMETER OutFile Write the response body to this path instead of returning it. .PARAMETER MaxRetries Maximum transient-failure retries. .PARAMETER IgnoreCertificateErrors Bypass TLS certificate validation. .OUTPUTS PSCustomObject: StatusCode [int], Body [string] ($null when -OutFile was used). #> [CmdletBinding()] param( [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] [string] $Uri, [Parameter(Mandatory = $false)] [ValidateSet('GET','POST','PUT','DELETE','PATCH')] [string] $Method = 'GET', [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] [string] $AuthString, [Parameter(Mandatory = $false)] [object] $Body, [Parameter(Mandatory = $false)] [string] $ContentType = 'application/json', [Parameter(Mandatory = $false)] [ValidateRange(1, 3600)] [int] $TimeoutSec = 100, [Parameter(Mandatory = $false)] [string] $OutFile, [Parameter(Mandatory = $false)] [ValidateRange(0, 10)] [int] $MaxRetries = 3, [Parameter(Mandatory = $false)] [switch] $IgnoreCertificateErrors ) $client = Get-VSAHttpClient -IgnoreCertificateErrors $IgnoreCertificateErrors.IsPresent [int] $attempt = 0 while ($true) { $certBypassPushed = $false $request = $null $response = $null $cts = $null try { # On 5.1 the bypass is a process-wide policy that must be installed around the send; # on Core this is a no-op (the handler already carries the validator). if ($IgnoreCertificateErrors) { & $script:VSAPushCertBypass $certBypassPushed = $true } $request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::new($Method), $Uri) $request.Headers.TryAddWithoutValidation('Authorization', $AuthString) | Out-Null $content = New-VSAHttpContent -Body $Body -ContentType $ContentType if ($null -ne $content) { $request.Content = $content } $cts = [System.Threading.CancellationTokenSource]::new([TimeSpan]::FromSeconds($TimeoutSec)) [int] $status = 0 [string] $responseBody = $null $transportError = $null try { $response = $client.SendAsync($request, $cts.Token).GetAwaiter().GetResult() $status = [int]$response.StatusCode if ($OutFile -and $response.IsSuccessStatusCode) { [System.IO.File]::WriteAllBytes($OutFile, $response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult()) } else { $responseBody = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() } } catch { # No HTTP response at all: reset socket, DNS failure, or the timeout token firing. $transportError = $_.Exception $status = 0 } if ((Test-VSARetryable -StatusCode $status -Method $Method -NoResponse ($null -ne $transportError)) -and $attempt -lt $MaxRetries) { $attempt++ $waitSeconds = Get-VSABackoffSeconds -Attempt $attempt -RetryAfterSeconds (Get-VSARetryAfterSeconds -Response $response) $statusLabel = if ($status -eq 0) { 'no response' } else { "HTTP $status ($(Get-VSAHttpStatusName -StatusCode $status))" } Write-Warning "Transient $statusLabel on $Method $Uri. Retry $attempt of $MaxRetries in $waitSeconds second(s)..." if ($waitSeconds -gt 0) { Start-Sleep -Seconds $waitSeconds } continue } if ($null -ne $transportError -or $status -ge 400) { throw (New-VSATransportError -StatusCode $status -Method $Method -Uri $Uri -Body $responseBody -InnerException $transportError) } return [pscustomobject]@{ StatusCode = $status; Body = $responseBody } } finally { if ($null -ne $response) { $response.Dispose() } if ($null -ne $request) { $request.Dispose() } if ($null -ne $cts) { $cts.Dispose() } if ($certBypassPushed) { & $script:VSAPopCertBypass } } } } |