src/Invoke-AZTIGraphRequest.ps1
|
#Requires -Version 7.0 Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' if (-not (Get-Command Invoke-ScoutDiagnosticOperation -ErrorAction SilentlyContinue)) { . (Join-Path $PSScriptRoot 'Write-AZTIRunLog.ps1') } if (-not (Get-Command Get-ScoutHttpFailure -ErrorAction SilentlyContinue)) { . (Join-Path $PSScriptRoot 'Get-ScoutHttpFailure.ps1') } <# .Synopsis Execute a Microsoft Graph REST API request with automatic pagination and throttle handling. .DESCRIPTION Wrapper around Invoke-RestMethod or Microsoft.Graph.Authentication for Microsoft Graph API calls. Automatically: - Obtains a bearer token via Get-AZSCGraphToken - Builds the full URL from a relative path (e.g. /v1.0/users) - Follows @odata.nextLink for multi-page responses - Handles HTTP 429 (Too Many Requests) with Retry-After header - Returns the aggregated .value array (or raw response for single-object queries) NO Microsoft.Graph SDK dependency — uses Invoke-RestMethod directly. .PARAMETER Uri Relative Graph API path, e.g. '/v1.0/users' or '/beta/identity/conditionalAccess/policies'. A full URL (https://graph.microsoft.com/...) is also accepted. .PARAMETER Method HTTP method. Default: GET. .PARAMETER Body Request body for POST/PATCH/PUT requests. Will be serialized to JSON if a hashtable/PSObject. .PARAMETER SinglePage Do not follow @odata.nextLink — return only the first page of results. .PARAMETER MaxRetries Maximum number of retries for transient errors (429, 5xx). Default: 5. .PARAMETER SuppressFailureWarning Do not write the final warning before rethrowing a failed request. Intended for callers such as the permission audit that catch the exception and emit a more specific structured result. .PARAMETER TenantID Optional tenant ID to scope the underlying Graph token to. See Get-AZSCGraphToken -- without this, the token comes from az CLI's ambient default tenant, which is not necessarily the tenant being audited or collected against. .OUTPUTS [PSObject[]] Aggregated .value array, or the raw response for single-object endpoints. .LINK https://github.com/Hybrid-Solutions-Cloud/azure-scout .COMPONENT This PowerShell Module is part of Azure Scout (AZSC) .NOTES Version: 1.0.0 Authors: thisismydemo #> function Invoke-AZSCGraphRequest { [CmdletBinding()] param( [Parameter(Mandatory)] [string]$Uri, [ValidateSet('GET', 'POST', 'PATCH', 'PUT', 'DELETE')] [string]$Method = 'GET', [object]$Body, [switch]$SinglePage, [int]$MaxRetries = 5, [string]$TenantID, [string[]]$RequiredScopes = @(), [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud')] [string]$AzureEnvironment, [switch]$SuppressFailureWarning ) if (-not $AzureEnvironment) { try { $azContext = Get-AzContext -ErrorAction SilentlyContinue if ($azContext -and $azContext.PSObject.Properties.Name -contains 'Environment' -and $azContext.Environment -and $azContext.Environment.PSObject.Properties.Name -contains 'Name') { $AzureEnvironment = [string]$azContext.Environment.Name } } catch { } } if ($AzureEnvironment -notin @('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud')) { $AzureEnvironment = 'AzureCloud' } $baseUrl = switch ($AzureEnvironment) { 'AzureUSGovernment' { 'https://graph.microsoft.us' } 'AzureChinaCloud' { 'https://microsoftgraph.chinacloudapi.cn' } default { 'https://graph.microsoft.com' } } # Normalise URI — accept both relative ("/v1.0/users") and absolute URLs if ($Uri -notmatch '^https?://') { $fullUri = "$baseUrl$Uri" } else { $fullUri = $Uri } Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + " - Graph $Method $fullUri") $allResults = [System.Collections.Generic.List[object]]::new() $currentUri = $fullUri do { $headers = Get-AZSCGraphToken -TenantID $TenantID -AzureEnvironment $AzureEnvironment -Scopes $RequiredScopes $useMgGraph = ($headers['X-AzureScout-GraphProvider'] -eq 'Microsoft.Graph.Authentication') $requestParams = @{ Uri = $currentUri Method = $Method ErrorAction = 'Stop' } if ($useMgGraph) { $requestParams['OutputType'] = 'PSObject' } else { $requestParams['Headers'] = $headers } if ($Body) { if ($Body -is [string]) { $requestParams['Body'] = $Body } else { $requestParams['Body'] = $Body | ConvertTo-Json -Depth 20 } $requestParams['ContentType'] = 'application/json' } $response = $null $retryCount = 0 while ($retryCount -le $MaxRetries) { try { $response = if ($useMgGraph) { Invoke-ScoutDiagnosticOperation -Operation { Invoke-MgGraphRequest @requestParams } } else { Invoke-ScoutDiagnosticOperation -Operation { Invoke-RestMethod @requestParams } } break } catch { $failure = Get-ScoutHttpFailure -ErrorRecord $_ $statusCode = $failure.StatusCode if (Get-Command Write-AZSCLog -ErrorAction SilentlyContinue) { Write-AZSCLog -Level ERROR -Message "Graph $Method $currentUri : $($failure.Summary)" } # Handle 429 (throttled) and 5xx (transient server errors) if ($statusCode -eq 429 -or ($statusCode -ge 500 -and $statusCode -lt 600)) { $retryCount++ # 504 Gateway Timeout means Graph already spent its full server-side budget # (about two minutes) on this exact query. Repeating it five times repeats the # same wait: a live nine-subscription scan lost 12 minutes this way on the # 30-day sign-in log query. One retry covers a transient gateway fault. $gatewayTimeoutExhausted = ($statusCode -eq 504 -and $retryCount -gt 1) if ($retryCount -gt $MaxRetries -or $gatewayTimeoutExhausted) { if ($gatewayTimeoutExhausted) { $MaxRetries = $retryCount - 1 } if (-not $SuppressFailureWarning) { Write-Warning "Graph API request failed after $MaxRetries retries: $($_.Exception.Message)" } throw } # Use Retry-After header if present, otherwise exponential backoff $retryAfter = $null if ($_.Exception.PSObject.Properties['Response'] -and $_.Exception.Response -and $_.Exception.Response.PSObject.Properties['Headers']) { try { $retryAfterValues = $_.Exception.Response.Headers.GetValues('Retry-After') if ($retryAfterValues) { $retryAfter = [int]$retryAfterValues[0] } } catch { # Header not present — use backoff Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + ' - Retry-After header not present: ' + $_.Exception.Message) } } if (-not $retryAfter -or $retryAfter -lt 1) { $retryAfter = [math]::Pow(2, $retryCount) } Write-Warning "Graph API returned $statusCode. Retrying in $retryAfter seconds (attempt $retryCount/$MaxRetries)..." Start-Sleep -Seconds $retryAfter # Refresh token in case it expired during wait $headers = Get-AZSCGraphToken -TenantID $TenantID -AzureEnvironment $AzureEnvironment -Scopes $RequiredScopes $useMgGraph = ($headers['X-AzureScout-GraphProvider'] -eq 'Microsoft.Graph.Authentication') if ($useMgGraph) { $requestParams.Remove('Headers') $requestParams['OutputType'] = 'PSObject' } else { $requestParams.Remove('OutputType') $requestParams['Headers'] = $headers } } else { # Non-retryable error — propagate if (-not $SuppressFailureWarning) { Write-Warning "Graph API request failed: $($_.Exception.Message)" } throw } } } # Collect results if ($null -ne $response) { if ($response.PSObject.Properties.Name -contains 'value') { # Collection endpoint — accumulate .value items foreach ($item in $response.value) { $allResults.Add($item) } } else { # Single-object endpoint — return as-is return $response } } # Pagination $currentUri = $null if (-not $SinglePage -and $null -ne $response -and $response.PSObject.Properties.Name -contains '@odata.nextLink') { $currentUri = $response.'@odata.nextLink' Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + " - Following nextLink (collected $($allResults.Count) items so far)") } } while ($currentUri) Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + " - Graph request complete. Total items: $($allResults.Count)") return $allResults.ToArray() } <# .SYNOPSIS Reads several Microsoft Graph collection URIs at the same time. .DESCRIPTION Sends the first page of every URI concurrently, then completes each one in order through Invoke-AZSCGraphRequest: a URI whose first page failed is re-read from the start (so it gets the normal retry handling and error), and a URI with more pages is followed from its nextLink. Rows are returned in the order the URIs were given. Used for datasets split into time windows. On a live tenant five week-long sign-in log windows took 34 to 116 seconds one after another and about 13 seconds together. Falls back to reading every URI one after another when no bearer token is available to hand to the worker threads (for example when the Microsoft.Graph.Authentication provider is in use). .PARAMETER Uri Relative ("/v1.0/...") or absolute Graph URIs, GET only. #> function Invoke-AZSCGraphRequestParallel { [CmdletBinding()] [OutputType([object[]])] param( [Parameter(Mandatory)] [string[]]$Uri, [string]$TenantID, [string[]]$RequiredScopes = @(), [ValidateRange(1, 16)] [int]$ThrottleLimit = 5, [int]$TimeoutSec = 150 ) $rows = [System.Collections.Generic.List[object]]::new() $serialRead = { param([string]$TargetUri) foreach ($row in @(Invoke-AZSCGraphRequest -Uri $TargetUri -TenantID $TenantID -RequiredScopes $RequiredScopes)) { if ($null -ne $row) { $rows.Add($row) } } } $headers = Get-AZSCGraphToken -TenantID $TenantID -Scopes $RequiredScopes $authorization = if ($headers -is [System.Collections.IDictionary] -and $headers.Contains('Authorization')) { [string]$headers['Authorization'] } else { '' } $usesMgGraph = ($headers -is [System.Collections.IDictionary] -and $headers.Contains('X-AzureScout-GraphProvider') -and $headers['X-AzureScout-GraphProvider'] -eq 'Microsoft.Graph.Authentication') if ($Uri.Count -lt 2 -or $usesMgGraph -or [string]::IsNullOrWhiteSpace($authorization)) { foreach ($target in $Uri) { & $serialRead $target } return $rows.ToArray() } $environmentName = 'AzureCloud' try { $azContext = Get-AzContext -ErrorAction SilentlyContinue if ($azContext -and $azContext.PSObject.Properties.Name -contains 'Environment' -and $azContext.Environment -and $azContext.Environment.PSObject.Properties.Name -contains 'Name') { $environmentName = [string]$azContext.Environment.Name } } catch { Write-Debug "Invoke-AZSCGraphRequestParallel: Azure context not readable, assuming AzureCloud: $($_.Exception.Message)" } $baseUrl = switch ($environmentName) { 'AzureUSGovernment' { 'https://graph.microsoft.us' } 'AzureChinaCloud' { 'https://microsoftgraph.chinacloudapi.cn' } default { 'https://graph.microsoft.com' } } $index = 0 $work = foreach ($target in $Uri) { [pscustomobject]@{ Index = $index++ Uri = $target FullUri = if ($target -match '^https?://') { $target } else { "$baseUrl$target" } } } $timer = [System.Diagnostics.Stopwatch]::StartNew() $firstPages = @{} try { $work | ForEach-Object -ThrottleLimit $ThrottleLimit -Parallel { $item = $_ try { $response = Invoke-RestMethod -Uri $item.FullUri -Method GET -Headers @{ Authorization = $using:authorization } -TimeoutSec $using:TimeoutSec -ErrorAction Stop $hasValue = ($null -ne $response -and $response.PSObject.Properties.Name -contains 'value') $hasNext = ($null -ne $response -and $response.PSObject.Properties.Name -contains '@odata.nextLink') [pscustomobject]@{ Index = $item.Index Rows = if ($hasValue) { @($response.value) } elseif ($null -ne $response) { @($response) } else { @() } NextLink = if ($hasNext) { [string]$response.'@odata.nextLink' } else { '' } Error = '' } } catch { [pscustomobject]@{ Index = $item.Index; Rows = @(); NextLink = ''; Error = [string]$_.Exception.Message } } } | ForEach-Object { $firstPages[[int]$_.Index] = $_ } } catch { Write-Debug "Invoke-AZSCGraphRequestParallel: concurrent read unavailable, reading one at a time: $($_.Exception.Message)" $firstPages = @{} } Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + " - Graph concurrent first pages: $($firstPages.Count) of $(@($work).Count) in $($timer.ElapsedMilliseconds) ms") foreach ($item in $work) { $page = if ($firstPages.ContainsKey([int]$item.Index)) { $firstPages[[int]$item.Index] } else { $null } if ($null -eq $page -or -not [string]::IsNullOrEmpty($page.Error)) { if ($null -ne $page) { Write-Debug "Invoke-AZSCGraphRequestParallel: first page failed for $($item.FullUri), re-reading with retries: $($page.Error)" } & $serialRead $item.Uri continue } foreach ($row in @($page.Rows)) { if ($null -ne $row) { $rows.Add($row) } } if (-not [string]::IsNullOrEmpty($page.NextLink)) { & $serialRead $page.NextLink } } return $rows.ToArray() } |