Modules/businessdev.ALbuild.Marketplace/Private/Send-BcIngestionBlob.ps1
|
function Send-BcIngestionBlob { <# .SYNOPSIS Uploads a file to the pre-signed URI Partner Center issued for a package, using that URI exactly as it was issued. .DESCRIPTION Replaces the Az.Storage upload that broke every AppSource release on 2026-09-01 (releases 325/326). The old code took the `fileSasUri`, kept only the first DNS label as a storage-account name and let `New-AzStorageContext -StorageAccountName` rebuild the endpoint - which hard-codes '<account>.blob.core.windows.net'. Partner Center now hands the upload target out through Azure Front Door: issued : workloadfileeus1-<id>.b02.azurefd.net/<product>/pkgfile-<id>/<blob> rebuilt : workloadfileeus1-<id>.blob.core.windows.net <- does not resolve DNS failed on the rebuilt host, the storage client retried until its budget was exhausted, and after ~11 minutes the task died with a bare 'An error occurred while sending the request.' The SAS is a USER-DELEGATION SAS (skoid/sktid/skv in the query) and is signed over host *and* path, so the endpoint can never be reconstructed - it has to be used verbatim. That also removes the Az.Storage dependency from the release agent altogether. Small files go up as a single Put Blob; anything larger is uploaded as Put Block + Put Block List, which keeps every individual request small enough for the Front Door hop. Each request has an explicit timeout and is retried on a transient failure. A failure reports the method, the endpoint (host + path, never the SAS signature), the HTTP status, the x-ms-request-id and the response body, so the next incident is diagnosable from the pipeline log alone. .PARAMETER SasUri The full `fileSasUri` from the ingestion API's package response. Used verbatim. .PARAMETER Path The local file to upload. .PARAMETER BlockSizeMB Block size for the chunked path, and the threshold below which a single Put Blob is used. Default 4 MB - comfortably inside the request limits of the Front Door hop. .PARAMETER TimeoutSeconds Per-request timeout. Default 300. .PARAMETER MaxAttempts Attempts per request before giving up. Default 4. .OUTPUTS PSCustomObject: Endpoint, SizeBytes, Blocks, Seconds, ThroughputMBs. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Called from New-BcMarketplaceSubmission, which owns the ShouldProcess gate.')] [CmdletBinding()] [OutputType([PSCustomObject])] param( [Parameter(Mandatory)] [string] $SasUri, [Parameter(Mandatory)] [string] $Path, [int] $BlockSizeMB = 4, [int] $TimeoutSeconds = 300, [int] $MaxAttempts = 4 ) if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "Upload source not found: '$Path'." } if ($SasUri -notmatch '\?') { throw 'The package upload URI carries no SAS query string; Partner Center did not return a usable fileSasUri.' } $uri = [System.Uri] $SasUri # Log target: host + path only. The query holds the SAS signature and must never reach a build log. $endpoint = "$($uri.Scheme)://$($uri.Host)$($uri.AbsolutePath)" $blockSize = $BlockSizeMB * 1MB $size = (Get-Item -LiteralPath $Path).Length $name = Split-Path -Leaf $Path Write-ALbuildLog " upload target : $endpoint" Write-ALbuildLog " payload : $name ($([math]::Round($size / 1MB, 2)) MB)" $started = Get-Date $blocks = 0 if ($size -le $blockSize) { Write-ALbuildLog ' strategy : single Put Blob' $bytes = [System.IO.File]::ReadAllBytes((Resolve-Path -LiteralPath $Path).ProviderPath) Invoke-BcBlobRequest -Uri $SasUri -Method Put -Body $bytes -Endpoint $endpoint ` -Headers @{ 'x-ms-blob-type' = 'BlockBlob'; 'Content-Type' = 'application/octet-stream' } ` -TimeoutSeconds $TimeoutSeconds -MaxAttempts $MaxAttempts -Operation "upload '$name'" | Out-Null } else { $total = [math]::Ceiling($size / $blockSize) Write-ALbuildLog " strategy : $total block(s) of $BlockSizeMB MB (Put Block + Put Block List)" $blockIds = New-Object System.Collections.Generic.List[string] $stream = [System.IO.File]::OpenRead((Resolve-Path -LiteralPath $Path).ProviderPath) try { $buffer = New-Object byte[] $blockSize while (($read = $stream.Read($buffer, 0, $blockSize)) -gt 0) { $blocks++ # Block ids must all be the same length; a fixed-width ordinal keeps them ordered and unique. $blockId = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($blocks.ToString('D6'))) $blockIds.Add($blockId) $payload = if ($read -eq $blockSize) { $buffer } else { $buffer[0..($read - 1)] } $blockUri = "$SasUri&comp=block&blockid=$([System.Uri]::EscapeDataString($blockId))" Invoke-BcBlobRequest -Uri $blockUri -Method Put -Body $payload -Endpoint $endpoint ` -Headers @{ 'Content-Type' = 'application/octet-stream' } ` -TimeoutSeconds $TimeoutSeconds -MaxAttempts $MaxAttempts -Operation "upload block $blocks/$total of '$name'" | Out-Null Write-ALbuildLog -Level Verbose " block $blocks/$total ($read bytes) ok" } } finally { $stream.Dispose() } # Commit the blocks. The block list carries the RAW base64 ids (only the query needs escaping). $xml = '<?xml version="1.0" encoding="utf-8"?><BlockList>' + (($blockIds | ForEach-Object { "<Latest>$_</Latest>" }) -join '') + '</BlockList>' Invoke-BcBlobRequest -Uri "$SasUri&comp=blocklist" -Method Put -Body ([System.Text.Encoding]::UTF8.GetBytes($xml)) -Endpoint $endpoint ` -Headers @{ 'Content-Type' = 'application/xml' } ` -TimeoutSeconds $TimeoutSeconds -MaxAttempts $MaxAttempts -Operation "commit the block list for '$name'" | Out-Null } $seconds = [math]::Round(((Get-Date) - $started).TotalSeconds, 1) $throughput = if ($seconds -gt 0) { [math]::Round(($size / 1MB) / $seconds, 2) } else { 0 } Write-ALbuildLog " transferred : $([math]::Round($size / 1MB, 2)) MB in $seconds s ($throughput MB/s)" return [PSCustomObject]@{ Endpoint = $endpoint SizeBytes = $size Blocks = $blocks Seconds = $seconds ThroughputMBs = $throughput } } function Invoke-BcBlobRequest { # Internal: one blob-service request with an explicit timeout, transient retry and a diagnosable error. [CmdletBinding()] param( [Parameter(Mandatory)] [string] $Uri, [Parameter(Mandatory)] [ValidateSet('Put', 'Get', 'Head')] [string] $Method, [Parameter(Mandatory)] [string] $Endpoint, [Parameter(Mandatory)] [string] $Operation, [byte[]] $Body, [hashtable] $Headers = @{}, [int] $TimeoutSeconds = 300, [int] $MaxAttempts = 4 ) # Windows PowerShell 5.1 renders a progress bar per request, which costs more than the transfer itself. $ProgressPreference = 'SilentlyContinue' $attempt = 0 while ($true) { $attempt++ try { $params = @{ Uri = $Uri; Method = $Method; Headers = $Headers; TimeoutSec = $TimeoutSeconds; UseBasicParsing = $true; ErrorAction = 'Stop' } if ($null -ne $Body) { $params['Body'] = $Body } return Invoke-WebRequest @params } catch { # StrictMode-safe: an exception type without a .Response member must read as $null, not throw. $response = $null $respProp = $_.Exception.PSObject.Properties['Response'] if ($respProp) { $response = $respProp.Value } $status = 0 if ($response) { try { $status = [int] $response.StatusCode } catch { $status = 0 } } if (-not $status -and "$($_.Exception.Message)" -match '\((\d{3})\)') { $status = [int] $Matches[1] } # status 0 means the request never got an HTTP response at all (DNS, TCP, TLS, timeout) - which # is exactly the class of failure that took the releases down, so it is worth retrying. $transient = ($status -eq 0) -or ($status -in @(408, 429, 500, 502, 503, 504)) if ($transient -and $attempt -lt $MaxAttempts) { $why = if ($status) { "HTTP $status" } else { 'no response' } Write-ALbuildLog -Level Warning " $Operation failed ($why, attempt $attempt/$MaxAttempts): $(Format-BcErrorMessage -Text $_.Exception.Message). Retrying in $($attempt * 5)s..." Start-Sleep -Seconds ($attempt * 5) continue } $lines = New-Object System.Collections.Generic.List[string] $lines.Add("Failed to $Operation after $attempt attempt(s).") $lines.Add(" endpoint : $Method $Endpoint") $lines.Add(" status : $(if ($status) { $status } else { 'no HTTP response (DNS, TCP, TLS or timeout)' })") $requestId = Get-BcBlobResponseHeader -Response $response -Name 'x-ms-request-id' if ($requestId) { $lines.Add(" request id : $requestId") } $detail = Get-BcMarketplaceErrorDetail -ErrorRecord $_ if ($detail) { $lines.Add(" response : $(Format-BcErrorMessage -Text $detail)") } # The whole chain: the outer HttpRequestException message ('An error occurred while sending the # request.') is useless on its own - the reason always sits in an inner exception. $ex = $_.Exception; $depth = 0 while ($ex -and $depth -lt 6) { $lines.Add(" [$($ex.GetType().Name)] $($ex.Message)") $ex = $ex.InnerException; $depth++ } throw ($lines -join [System.Environment]::NewLine) } } } function Get-BcBlobResponseHeader { # Internal: read one response header across 5.1 (string values) and 7+ (string[] values). [CmdletBinding()] [OutputType([string])] param([AllowNull()] $Response, [Parameter(Mandatory)] [string] $Name) if (-not $Response) { return $null } try { $headersProp = $Response.PSObject.Properties['Headers'] if (-not $headersProp -or -not $headersProp.Value) { return $null } $value = $headersProp.Value[$Name] if ($null -eq $value) { return $null } if ($value -is [array]) { return "$($value -join ', ')" } return "$value" } catch { return $null } } |