Modules/businessdev.ALbuild.Feeds/Classes/BcNuGetFeedProvider.ps1
|
class BcNuGetFeedProvider { [string] $Name [string] $Url [string] $Token [string] $IdScheme # e.g. '{publisher}.{name}.symbols.{id}' [string] $Kind # symbols | apps | runtime [string] $CacheFolder # Target build the current resolution is aimed at. Set by the solver before FindCandidates so an # "indirect" metapackage can pick the runtime sub-package build closest to the target BC build. [string] $TargetPlatform [string] $TargetApplication hidden [string] $SearchQueryServiceUrl hidden [string] $PackageBaseAddressUrl hidden [string] $RegistrationsBaseUrl hidden [bool] $Initialized = $false # Per-resolve memoisation: the backtracking solver queries the same packages many times, so the # version list and registration (dependency) metadata for a package id are fetched once and reused. hidden [hashtable] $VersionCache = @{} hidden [hashtable] $RegistrationCache = @{} hidden [hashtable] $SearchCache = @{} BcNuGetFeedProvider([string] $name, [string] $url, [string] $token, [string] $idScheme, [string] $kind, [string] $cacheFolder) { $this.Name = $name $this.Url = $url $this.Token = $token $this.IdScheme = if ($idScheme) { $idScheme } else { '{publisher}.{name}.{id}' } $this.Kind = if ($kind) { $kind } else { 'symbols' } $this.CacheFolder = $cacheFolder } # Normalise a publisher/name segment for use in a NuGet package id. static [string] Normalize([string] $value) { if (-not $value) { return '' } return ($value -replace '[^A-Za-z0-9_\-]', '') } # Extract the AL app id (trailing GUID) from a BC NuGet package id, or '' when there is none # (e.g. Microsoft.Application/Microsoft.Platform, or a runtime sub-package whose id ends in a # dashed version). The convention is '{publisher}.{name}[.runtime].{appId}'. static [string] AppIdFromPackageId([string] $packageId) { if (-not $packageId) { return '' } $m = [regex]::Match($packageId, '([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$') if ($m.Success) { return $m.Value } return '' } # A human-readable name derived from a package id: the name segment(s) between the publisher and # the trailing app-id GUID, with a trailing 'runtime' marker dropped. Used only for friendlier # logs/output when no friendlier name is available (the segments are the normalised, space-less # publisher/name from the package id). E.g. 'ContiniaSoftware.ContiniaCore.<guid>' -> 'ContiniaCore'. static [string] NameFromPackageId([string] $packageId) { if (-not $packageId) { return '' } $guid = [BcNuGetFeedProvider]::AppIdFromPackageId($packageId) $core = if ($guid) { $packageId.Substring(0, $packageId.Length - $guid.Length).TrimEnd('.', '-') } else { $packageId } $parts = @($core -split '\.' | Where-Object { $_ }) if ($parts.Count -gt 1) { $parts = $parts[1..($parts.Count - 1)] } # drop the publisher segment $parts = @($parts | Where-Object { $_ -notmatch '^(?i)runtime$' }) if ($parts.Count -eq 0) { return $core } return ($parts -join '.') } # The lower bound of a NuGet version range ('[27.0.0, )', '27.0.0', '(,2.0.0)', ...). A bare # version is itself the inclusive minimum. Returns '0.0.0.0' when unbounded below. static [string] RangeLowerBound([string] $range) { if (-not $range) { return '0.0.0.0' } $r = $range.Trim() if ($r -notmatch '[\[\](),]') { return $r } $inner = $r.Trim('[', ']', '(', ')') $lower = ($inner -split ',', 2)[0].Trim() if (-not $lower) { return '0.0.0.0' } return $lower } [hashtable] GetHeaders() { $headers = @{ 'Content-Type' = 'application/json; charset=utf-8' } if ($this.Token -and ($this.Url -notlike 'https://api.nuget.org/*')) { $basic = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("user:$($this.Token)")) $headers['Authorization'] = "Basic $basic" } return $headers } hidden [void] EnsureInitialized() { if ($this.Initialized) { return } $index = Invoke-RestMethod -Uri $this.Url -Headers ($this.GetHeaders()) -Method Get -UseBasicParsing -TimeoutSec 60 -ErrorAction Stop foreach ($resource in $index.resources) { switch -Wildcard ($resource.'@type') { 'SearchQueryService*' { if (-not $this.SearchQueryServiceUrl) { $this.SearchQueryServiceUrl = $resource.'@id' } } 'PackageBaseAddress/3.0.0' { $this.PackageBaseAddressUrl = $resource.'@id' } 'RegistrationsBaseUrl/3.6.0' { $this.RegistrationsBaseUrl = $resource.'@id' } 'RegistrationsBaseUrl' { if (-not $this.RegistrationsBaseUrl) { $this.RegistrationsBaseUrl = $resource.'@id' } } } } if (-not $this.PackageBaseAddressUrl) { throw "NuGet feed '$($this.Name)' does not expose PackageBaseAddress/3.0.0." } $this.Initialized = $true } [string] MapPackageId([string] $appId, [string] $name, [string] $publisher) { $id = $this.IdScheme $id = $id.Replace('{publisher}', [BcNuGetFeedProvider]::Normalize($publisher)) $id = $id.Replace('{name}', [BcNuGetFeedProvider]::Normalize($name)) $id = $id.Replace('{id}', $appId) $id = $id -replace '\.\.', '.' return ($id.Trim('.')) } # Discover a package id on this feed by its AL app id (trailing GUID), via the search service. # The app id is the only stable key: publishers vary the publisher/name segments between a # package and its dependants' nuspec references, so name-based reconstruction is unreliable. # Returns the id of the app/metapackage (one ending in '.{appId}'), or '' if none is found. [string] SearchPackageId([string] $appId) { $this.EnsureInitialized() if ($this.SearchCache.ContainsKey($appId)) { return $this.SearchCache[$appId] } $found = '' if ($this.SearchQueryServiceUrl) { try { $searchUrl = "$($this.SearchQueryServiceUrl)?q=$([uri]::EscapeDataString($appId))&take=50&prerelease=true" $response = Invoke-RestMethod -Uri $searchUrl -Headers ($this.GetHeaders()) -Method Get -UseBasicParsing -TimeoutSec 60 -ErrorAction Stop foreach ($item in @($response.data)) { if ([BcNuGetFeedProvider]::AppIdFromPackageId("$($item.id)") -ieq $appId) { $found = "$($item.id)"; break } } } catch { $found = '' } } $this.SearchCache[$appId] = $found return $found } # The literal text this feed's id scheme places between the name and the app id (e.g. '.runtime.' # for '{publisher}.{name}.runtime.{id}', '.' for the plain scheme). [string] SchemeInfix() { $m = [regex]::Match($this.IdScheme, '\{name\}(.*?)\{id\}') if ($m.Success) { return $m.Groups[1].Value } return '.' } # Rewrite a '{publisher}.{name}.{id}'-style package id (e.g. read verbatim from a nuspec) to this # feed's id scheme by inserting the scheme infix before the trailing app-id GUID. Returns '' when # the id has no trailing GUID or already matches the scheme. [string] AdjustHintToScheme([string] $hint) { $guid = [BcNuGetFeedProvider]::AppIdFromPackageId($hint) if (-not $guid) { return '' } $prefix = $hint.Substring(0, $hint.Length - $guid.Length).TrimEnd('.', '-') $adjusted = "$prefix$($this.SchemeInfix())$guid" if ($adjusted -eq $hint) { return '' } return $adjusted } [string[]] GetVersions([string] $packageId) { if ($this.VersionCache.ContainsKey($packageId)) { return $this.VersionCache[$packageId] } $this.EnsureInitialized() $requestUrl = "$($this.PackageBaseAddressUrl.TrimEnd('/'))/$($packageId.ToLowerInvariant())/index.json" $versions = @() try { $response = Invoke-RestMethod -Uri $requestUrl -Headers ($this.GetHeaders()) -Method Get -UseBasicParsing -TimeoutSec 60 -ErrorAction Stop $versions = @($response.versions) } catch { $versions = @() } $this.VersionCache[$packageId] = $versions return $versions } # Read every published version of a package together with its declared NuGet dependencies, from # the registration index - no .app download required. Returns @( @{ Version; Deps = @(@{Id;Range}) } ). # This is the authoritative dependency source for BC NuGet packages: their nuspec/registration # carries the real app graph (including Microsoft.Application/Microsoft.Platform as the BC version # constraint and, for "indirect" metapackages, a pointer to the runtime sub-package). [object[]] GetRegistrations([string] $packageId) { if ($this.RegistrationCache.ContainsKey($packageId)) { return $this.RegistrationCache[$packageId] } $this.EnsureInitialized() if (-not $this.RegistrationsBaseUrl) { return @() } $idLower = $packageId.ToLowerInvariant() $regUrl = "$($this.RegistrationsBaseUrl.TrimEnd('/'))/$idLower/index.json" $result = [System.Collections.Generic.List[object]]::new() try { $index = Invoke-RestMethod -Uri $regUrl -Headers ($this.GetHeaders()) -Method Get -UseBasicParsing -TimeoutSec 60 -ErrorAction Stop } catch { $this.RegistrationCache[$packageId] = @() return @() } foreach ($page in @($index.items)) { $leaves = $page.items if (-not $leaves) { try { $leaves = (Invoke-RestMethod -Uri $page.'@id' -Headers ($this.GetHeaders()) -Method Get -UseBasicParsing -TimeoutSec 60 -ErrorAction Stop).items } catch { $leaves = @() } } foreach ($leaf in @($leaves)) { $entry = $leaf.catalogEntry if (-not $entry) { continue } if ($entry -is [string]) { try { $entry = Invoke-RestMethod -Uri $entry -Headers ($this.GetHeaders()) -Method Get -UseBasicParsing -TimeoutSec 60 -ErrorAction Stop } catch { continue } } $deps = [System.Collections.Generic.List[object]]::new() if ($entry.PSObject.Properties.Name -contains 'dependencyGroups') { foreach ($group in @($entry.dependencyGroups)) { if ($group.PSObject.Properties.Name -notcontains 'dependencies') { continue } foreach ($d in @($group.dependencies)) { $deps.Add(@{ Id = "$($d.id)"; Range = "$($d.range)" }) } } } $result.Add(@{ Version = "$($entry.version)"; Deps = $deps.ToArray() }) } } $registrations = $result.ToArray() $this.RegistrationCache[$packageId] = $registrations return $registrations } # The highest build that does not exceed the target BC platform (closest match from below, as # required for on-premise targets). When no target is given, the latest build is returned. # Returns '' when no build is at or below the target. Pure: the network-free core of the runtime # sub-package selection, so it is directly unit-testable. static [string] SelectClosestBuild([string[]] $builds, [string] $targetPlatform) { $list = @($builds | Where-Object { $_ }) if ($list.Count -eq 0) { return '' } $sorted = @($list | Sort-Object -Property @{ Expression = { ConvertTo-BcVersion $_ } }) if (-not $targetPlatform) { return ($sorted | Select-Object -Last 1) } $compatible = @($sorted | Where-Object { (Compare-BcVersion $_ $targetPlatform) -le 0 }) if ($compatible.Count -eq 0) { return '' } return ($compatible | Select-Object -Last 1) } # The runtime build to download for this target: queries the sub-package's published builds and # picks the closest match at or below the target platform. [string] ClosestRuntimeBuild([string] $runtimePackageId) { return [BcNuGetFeedProvider]::SelectClosestBuild($this.GetVersions($runtimePackageId), $this.TargetPlatform) } # Build a single candidate manifest from a registration entry, classifying its dependencies and # resolving any runtime indirection. Returns $null when the candidate is not installable on the # target build (e.g. an indirect package with no runtime build at or below the target platform). hidden [object] BuildCandidate([string] $appId, [string] $packageId, [object] $entry) { $manifest = [BcPackageManifest]::new() $manifest.Id = $appId $manifest.Name = [BcNuGetFeedProvider]::NameFromPackageId($packageId) $manifest.Version = "$($entry.Version)" $manifest.Kind = $this.Kind $manifest.PackageId = $packageId $manifest.ProviderName = $this.Name $manifest.Provider = $this $deps = [System.Collections.Generic.List[BcPackageDependency]]::new() $runtimePointer = '' foreach ($d in @($entry.Deps)) { $depId = "$($d.Id)" $min = [BcNuGetFeedProvider]::RangeLowerBound("$($d.Range)") if ($depId -match '^(?i)microsoft\.application$') { $manifest.Application = $min; continue } if ($depId -match '^(?i)microsoft\.platform$') { $manifest.Platform = $min; continue } $depAppId = [BcNuGetFeedProvider]::AppIdFromPackageId($depId) if (-not $depAppId) { # No trailing GUID and not Application/Platform: an indirect runtime sub-package # pointer, recognised by its trailing dashed app-version (e.g. '...-18-2-459-25417'). # Identifying it by the dashed version - not by a shared name prefix - tolerates the # publisher-side naming drift between a metapackage and its runtime sub-package. if (-not ($depId -match '^(?i)microsoft\.') -and $depId -match '-\d+(-\d+)+$') { $runtimePointer = $depId } continue } $dep = [BcPackageDependency]::new($depAppId, [BcNuGetFeedProvider]::NameFromPackageId($depId), '', $min) $dep.PackageId = $depId if ($depId -match '^(?i)microsoft\.') { $dep.Publisher = 'Microsoft' } $deps.Add($dep) } $manifest.Dependencies = $deps.ToArray() if ($runtimePointer) { # Skip the (networked) runtime-build lookup for app versions the target build already rules # out on its required BC version - the solver would discard them anyway. $tooNewForApp = $this.TargetApplication -and $manifest.Application -and (Compare-BcVersion $manifest.Application $this.TargetApplication) -gt 0 $tooNewForPlatform = $this.TargetPlatform -and $manifest.Platform -and (Compare-BcVersion $manifest.Platform $this.TargetPlatform) -gt 0 if (-not ($tooNewForApp -or $tooNewForPlatform)) { $build = $this.ClosestRuntimeBuild($runtimePointer) if (-not $build) { return $null } $manifest.DownloadId = $runtimePointer $manifest.DownloadVersion = $build } } return $manifest } # Downloads and extracts a package, returning the path to the contained .app file (cached). [string] DownloadAppFile([string] $packageId, [string] $version) { $this.EnsureInitialized() $idLower = $packageId.ToLowerInvariant() $verLower = $version.ToLowerInvariant() $packageFolder = Join-Path $this.CacheFolder (Join-Path $idLower $verLower) if (-not (Test-Path -LiteralPath (Join-Path $packageFolder '.complete'))) { $requestUrl = "$($this.PackageBaseAddressUrl.TrimEnd('/'))/$idLower/$verLower/$idLower.$verLower.nupkg" $tempZip = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), [Guid]::NewGuid().ToString() + '.nupkg.zip') try { Invoke-WebRequest -Uri $requestUrl -Headers ($this.GetHeaders()) -OutFile $tempZip -UseBasicParsing -TimeoutSec 600 -ErrorAction Stop if (Test-Path -LiteralPath $packageFolder) { Remove-Item -LiteralPath $packageFolder -Recurse -Force } New-Item -Path $packageFolder -ItemType Directory -Force | Out-Null Expand-Archive -LiteralPath $tempZip -DestinationPath $packageFolder -Force Set-Content -LiteralPath (Join-Path $packageFolder '.complete') -Value (Get-Date -Format 'o') -Encoding UTF8 } finally { if (Test-Path -LiteralPath $tempZip) { Remove-Item -LiteralPath $tempZip -Force -ErrorAction SilentlyContinue } } } $apps = @(Get-ChildItem -LiteralPath $packageFolder -Filter '*.app' -Recurse -File) if ($apps.Count -eq 0) { throw "Package '$packageId' $version does not contain an .app file." } # A package can carry both a regular and a runtime app (named '*.runtime.app' or '*_Runtime.app'). # Prefer the runtime app for runtime feeds, the regular app otherwise; fall back to whatever # single app is present. $isRuntimeApp = { param($n) $n -match '(?i)(\.|_)runtime\.app$' } $preferRuntime = ($this.Kind -eq 'runtime') $appFile = $apps | Sort-Object -Property @{ Expression = { $rt = [bool](& $isRuntimeApp $_.Name) if ($preferRuntime) { if ($rt) { 0 } else { 1 } } else { if ($rt) { 1 } else { 0 } } } }, Name | Select-Object -First 1 return $appFile.FullName } # Solver contract: return candidate manifests (app versions >= minVersion) with real dependencies, # read from registration metadata. packageIdHints carries the exact package id(s) a dependency was # declared with (from a parent's nuspec); when present they are used verbatim, which is more # reliable than reconstructing the id from a normalised publisher/name. [BcPackageManifest[]] FindCandidates([string] $appId, [string] $name, [string] $publisher, [string] $minVersion, [string[]] $packageIdHints) { $packageIds = [System.Collections.Generic.List[string]]::new() foreach ($hint in @($packageIdHints)) { if (-not $hint) { continue } if (-not $packageIds.Contains($hint)) { $packageIds.Add($hint) } # A nuspec frequently states a dependency with the plain '{publisher}.{name}.{id}' id even # on a feed whose ids carry an infix (e.g. '.runtime.'). Also probe the hint rewritten to # this feed's id scheme so the dependency is found where it actually lives. $adjusted = $this.AdjustHintToScheme($hint) if ($adjusted -and -not $packageIds.Contains($adjusted)) { $packageIds.Add($adjusted) } } $mapped = $this.MapPackageId($appId, $name, $publisher) if ($mapped -and -not $packageIds.Contains($mapped)) { $packageIds.Add($mapped) } # Last resort: discover the id by searching the feed for the app id. Reliable across # publisher-side naming drift, but a network round-trip, so it is appended after the cheap, # deterministic id guesses. $discovered = $this.SearchPackageId($appId) if ($discovered -and -not $packageIds.Contains($discovered)) { $packageIds.Add($discovered) } $candidates = [System.Collections.Generic.List[BcPackageManifest]]::new() $sawRegistrations = $false foreach ($packageId in $packageIds) { $registrations = $this.GetRegistrations($packageId) if ($registrations.Count -eq 0) { continue } $sawRegistrations = $true foreach ($entry in $registrations) { if ((Compare-BcVersion $entry.Version $minVersion) -lt 0) { continue } $candidate = $this.BuildCandidate($appId, $packageId, $entry) if ($candidate) { # Prefer the caller's friendly name (e.g. from app.json) over the id-derived one. if ($name) { $candidate.Name = $name } $candidates.Add($candidate) } } } # Feeds that publish a parseable .app but expose no registration metadata: read the manifest. if (-not $sawRegistrations -and -not $this.RegistrationsBaseUrl) { foreach ($packageId in $packageIds) { foreach ($candidate in $this.FindCandidatesFromAppFile($appId, $packageId, $minVersion)) { $candidates.Add($candidate) } } } return $candidates.ToArray() } # Backward-compatible overload (no hints). [BcPackageManifest[]] FindCandidates([string] $appId, [string] $name, [string] $publisher, [string] $minVersion) { return $this.FindCandidates($appId, $name, $publisher, $minVersion, @()) } # Fallback candidate discovery for feeds that ship a parseable .app but no registration metadata: # download each version and read NavxManifest. Kept for compatibility; the registration path above # is preferred (and is the only thing that works for indirect/runtime packages). hidden [BcPackageManifest[]] FindCandidatesFromAppFile([string] $appId, [string] $packageId, [string] $minVersion) { $candidates = [System.Collections.Generic.List[BcPackageManifest]]::new() foreach ($version in $this.GetVersions($packageId)) { if ((Compare-BcVersion $version $minVersion) -lt 0) { continue } try { $appFile = $this.DownloadAppFile($packageId, $version) $info = Expand-BcAppFile -Path $appFile $manifest = [BcPackageManifest]::new() $manifest.Id = $appId $manifest.Name = "$($info.Name)" $manifest.Version = "$($info.Version)" $manifest.Dependencies = @($info.Dependencies | ForEach-Object { [BcPackageDependency]::new("$($_.Id)", "$($_.Name)", "$($_.Publisher)", "$($_.Version)") }) $manifest.Platform = "$($info.Platform)" $manifest.Application = "$($info.Application)" $manifest.Kind = $this.Kind $manifest.PackageId = $packageId $manifest.ProviderName = $this.Name $manifest.Provider = $this $candidates.Add($manifest) if ($info.ExtractedPath -and (Test-Path $info.ExtractedPath)) { Remove-Item $info.ExtractedPath -Recurse -Force -ErrorAction SilentlyContinue } } catch { Write-ALbuildLog -Level Warning "Skipping $packageId $version on feed '$($this.Name)': $($_.Exception.Message)" } } return $candidates.ToArray() } # Download contract: copy the resolved .app into the target folder, returning its path. [string] DownloadPackage([string] $packageId, [string] $version, [string] $targetFolder) { $appFile = $this.DownloadAppFile($packageId, $version) if (-not (Test-Path -LiteralPath $targetFolder)) { New-Item -Path $targetFolder -ItemType Directory -Force | Out-Null } $destination = Join-Path $targetFolder ([System.IO.Path]::GetFileName($appFile)) Copy-Item -LiteralPath $appFile -Destination $destination -Force return $destination } } |