src/Invoke-AZTIGraphRequest.ps1

#Requires -Version 7.0
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
if (-not (Get-Command Invoke-ScoutDiagnosticOperation -ErrorAction SilentlyContinue)) { . (Join-Path $PSScriptRoot 'Write-AZTIRunLog.ps1') }
if (-not (Get-Command Get-ScoutHttpFailure -ErrorAction SilentlyContinue)) { . (Join-Path $PSScriptRoot 'Get-ScoutHttpFailure.ps1') }

<#
.Synopsis
    Execute a Microsoft Graph REST API request with automatic pagination and throttle handling.
 
.DESCRIPTION
    Wrapper around Invoke-RestMethod or Microsoft.Graph.Authentication for Microsoft Graph API
    calls. Automatically:
      - Obtains a bearer token via Get-AZSCGraphToken
      - Builds the full URL from a relative path (e.g. /v1.0/users)
      - Follows @odata.nextLink for multi-page responses
      - Handles HTTP 429 (Too Many Requests) with Retry-After header
      - Returns the aggregated .value array (or raw response for single-object queries)
 
    NO Microsoft.Graph SDK dependency — uses Invoke-RestMethod directly.
 
.PARAMETER Uri
    Relative Graph API path, e.g. '/v1.0/users' or '/beta/identity/conditionalAccess/policies'.
    A full URL (https://graph.microsoft.com/...) is also accepted.
 
.PARAMETER Method
    HTTP method. Default: GET.
 
.PARAMETER Body
    Request body for POST/PATCH/PUT requests. Will be serialized to JSON if a hashtable/PSObject.
 
.PARAMETER SinglePage
    Do not follow @odata.nextLink — return only the first page of results.
 
.PARAMETER MaxRetries
    Maximum number of retries for transient errors (429, 5xx). Default: 5.
 
.PARAMETER SuppressFailureWarning
    Do not write the final warning before rethrowing a failed request. Intended for callers such
    as the permission audit that catch the exception and emit a more specific structured result.
 
.PARAMETER TenantID
    Optional tenant ID to scope the underlying Graph token to. See Get-AZSCGraphToken --
    without this, the token comes from az CLI's ambient default tenant, which is not
    necessarily the tenant being audited or collected against.
 
.OUTPUTS
    [PSObject[]] Aggregated .value array, or the raw response for single-object endpoints.
 
.LINK
    https://github.com/Hybrid-Solutions-Cloud/azure-scout
 
.COMPONENT
    This PowerShell Module is part of Azure Scout (AZSC)
 
.NOTES
    Version: 1.0.0
    Authors: thisismydemo
#>

function Invoke-AZSCGraphRequest {
    [CmdletBinding()]
    param(
        [Parameter(Mandatory)]
        [string]$Uri,

        [ValidateSet('GET', 'POST', 'PATCH', 'PUT', 'DELETE')]
        [string]$Method = 'GET',

        [object]$Body,

        [switch]$SinglePage,

        [int]$MaxRetries = 5,

        [string]$TenantID,
        [string[]]$RequiredScopes = @(),
        [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud')]
        [string]$AzureEnvironment,

        [switch]$SuppressFailureWarning
    )

    if (-not $AzureEnvironment) {
        try {
            $azContext = Get-AzContext -ErrorAction SilentlyContinue
            if ($azContext -and $azContext.PSObject.Properties.Name -contains 'Environment' -and
                $azContext.Environment -and $azContext.Environment.PSObject.Properties.Name -contains 'Name') {
                $AzureEnvironment = [string]$azContext.Environment.Name
            }
        }
        catch { }
    }
    if ($AzureEnvironment -notin @('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud')) {
        $AzureEnvironment = 'AzureCloud'
    }

    $baseUrl = switch ($AzureEnvironment) {
        'AzureUSGovernment' { 'https://graph.microsoft.us' }
        'AzureChinaCloud'   { 'https://microsoftgraph.chinacloudapi.cn' }
        default             { 'https://graph.microsoft.com' }
    }

    # Normalise URI — accept both relative ("/v1.0/users") and absolute URLs
    if ($Uri -notmatch '^https?://') {
        $fullUri = "$baseUrl$Uri"
    }
    else {
        $fullUri = $Uri
    }

    Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + " - Graph $Method $fullUri")

    $allResults = [System.Collections.Generic.List[object]]::new()
    $currentUri = $fullUri

    do {
        $headers = Get-AZSCGraphToken -TenantID $TenantID -AzureEnvironment $AzureEnvironment -Scopes $RequiredScopes

        $useMgGraph = ($headers['X-AzureScout-GraphProvider'] -eq 'Microsoft.Graph.Authentication')
        $requestParams = @{
            Uri         = $currentUri
            Method      = $Method
            ErrorAction = 'Stop'
        }
        if ($useMgGraph) {
            $requestParams['OutputType'] = 'PSObject'
        }
        else {
            $requestParams['Headers'] = $headers
        }

        if ($Body) {
            if ($Body -is [string]) {
                $requestParams['Body'] = $Body
            }
            else {
                $requestParams['Body'] = $Body | ConvertTo-Json -Depth 20
            }
            $requestParams['ContentType'] = 'application/json'
        }

        $response = $null
        $retryCount = 0

        while ($retryCount -le $MaxRetries) {
            try {
                $response = if ($useMgGraph) {
                    Invoke-ScoutDiagnosticOperation -Operation { Invoke-MgGraphRequest @requestParams }
                }
                else {
                    Invoke-ScoutDiagnosticOperation -Operation { Invoke-RestMethod @requestParams }
                }
                break
            }
            catch {
                $failure = Get-ScoutHttpFailure -ErrorRecord $_
                $statusCode = $failure.StatusCode
                if (Get-Command Write-AZSCLog -ErrorAction SilentlyContinue) {
                    Write-AZSCLog -Level ERROR -Message "Graph $Method $currentUri : $($failure.Summary)"
                }

                # Handle 429 (throttled) and 5xx (transient server errors)
                if ($statusCode -eq 429 -or ($statusCode -ge 500 -and $statusCode -lt 600)) {
                    $retryCount++
                    # 504 Gateway Timeout means Graph already spent its full server-side budget
                    # (about two minutes) on this exact query. Repeating it five times repeats the
                    # same wait: a live nine-subscription scan lost 12 minutes this way on the
                    # 30-day sign-in log query. One retry covers a transient gateway fault.
                    $gatewayTimeoutExhausted = ($statusCode -eq 504 -and $retryCount -gt 1)
                    if ($retryCount -gt $MaxRetries -or $gatewayTimeoutExhausted) {
                        if ($gatewayTimeoutExhausted) { $MaxRetries = $retryCount - 1 }
                        if (-not $SuppressFailureWarning) {
                            Write-Warning "Graph API request failed after $MaxRetries retries: $($_.Exception.Message)"
                        }
                        throw
                    }

                    # Use Retry-After header if present, otherwise exponential backoff
                    $retryAfter = $null
                    if ($_.Exception.PSObject.Properties['Response'] -and $_.Exception.Response -and $_.Exception.Response.PSObject.Properties['Headers']) {
                        try {
                            $retryAfterValues = $_.Exception.Response.Headers.GetValues('Retry-After')
                            if ($retryAfterValues) {
                                $retryAfter = [int]$retryAfterValues[0]
                            }
                        }
                        catch {
                            # Header not present — use backoff
                            Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + ' - Retry-After header not present: ' + $_.Exception.Message)
                        }
                    }

                    if (-not $retryAfter -or $retryAfter -lt 1) {
                        $retryAfter = [math]::Pow(2, $retryCount)
                    }

                    Write-Warning "Graph API returned $statusCode. Retrying in $retryAfter seconds (attempt $retryCount/$MaxRetries)..."
                    Start-Sleep -Seconds $retryAfter

                    # Refresh token in case it expired during wait
                    $headers = Get-AZSCGraphToken -TenantID $TenantID -AzureEnvironment $AzureEnvironment -Scopes $RequiredScopes
                    $useMgGraph = ($headers['X-AzureScout-GraphProvider'] -eq 'Microsoft.Graph.Authentication')
                    if ($useMgGraph) {
                        $requestParams.Remove('Headers')
                        $requestParams['OutputType'] = 'PSObject'
                    }
                    else {
                        $requestParams.Remove('OutputType')
                        $requestParams['Headers'] = $headers
                    }
                }
                else {
                    # Non-retryable error — propagate
                    if (-not $SuppressFailureWarning) {
                        Write-Warning "Graph API request failed: $($_.Exception.Message)"
                    }
                    throw
                }
            }
        }

        # Collect results
        if ($null -ne $response) {
            if ($response.PSObject.Properties.Name -contains 'value') {
                # Collection endpoint — accumulate .value items
                foreach ($item in $response.value) {
                    $allResults.Add($item)
                }
            }
            else {
                # Single-object endpoint — return as-is
                return $response
            }
        }

        # Pagination
        $currentUri = $null
        if (-not $SinglePage -and $null -ne $response -and $response.PSObject.Properties.Name -contains '@odata.nextLink') {
            $currentUri = $response.'@odata.nextLink'
            Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + " - Following nextLink (collected $($allResults.Count) items so far)")
        }

    } while ($currentUri)

    Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + " - Graph request complete. Total items: $($allResults.Count)")

    return $allResults.ToArray()
}

<#
.SYNOPSIS
    Reads several Microsoft Graph collection URIs at the same time.
 
.DESCRIPTION
    Sends the first page of every URI concurrently, then completes each one in order through
    Invoke-AZSCGraphRequest: a URI whose first page failed is re-read from the start (so it gets
    the normal retry handling and error), and a URI with more pages is followed from its
    nextLink. Rows are returned in the order the URIs were given.
 
    Used for datasets split into time windows. On a live tenant five week-long sign-in log
    windows took 34 to 116 seconds one after another and about 13 seconds together.
 
    Falls back to reading every URI one after another when no bearer token is available to hand
    to the worker threads (for example when the Microsoft.Graph.Authentication provider is in use).
 
.PARAMETER Uri
    Relative ("/v1.0/...") or absolute Graph URIs, GET only.
#>

function Invoke-AZSCGraphRequestParallel {
    [CmdletBinding()]
    [OutputType([object[]])]
    param(
        [Parameter(Mandatory)]
        [string[]]$Uri,

        [string]$TenantID,
        [string[]]$RequiredScopes = @(),

        [ValidateRange(1, 16)]
        [int]$ThrottleLimit = 5,

        [int]$TimeoutSec = 150
    )

    $rows = [System.Collections.Generic.List[object]]::new()
    $serialRead = {
        param([string]$TargetUri)
        foreach ($row in @(Invoke-AZSCGraphRequest -Uri $TargetUri -TenantID $TenantID -RequiredScopes $RequiredScopes)) {
            if ($null -ne $row) { $rows.Add($row) }
        }
    }

    $headers = Get-AZSCGraphToken -TenantID $TenantID -Scopes $RequiredScopes
    $authorization = if ($headers -is [System.Collections.IDictionary] -and $headers.Contains('Authorization')) { [string]$headers['Authorization'] } else { '' }
    $usesMgGraph = ($headers -is [System.Collections.IDictionary] -and $headers.Contains('X-AzureScout-GraphProvider') -and
        $headers['X-AzureScout-GraphProvider'] -eq 'Microsoft.Graph.Authentication')
    if ($Uri.Count -lt 2 -or $usesMgGraph -or [string]::IsNullOrWhiteSpace($authorization)) {
        foreach ($target in $Uri) { & $serialRead $target }
        return $rows.ToArray()
    }

    $environmentName = 'AzureCloud'
    try {
        $azContext = Get-AzContext -ErrorAction SilentlyContinue
        if ($azContext -and $azContext.PSObject.Properties.Name -contains 'Environment' -and
            $azContext.Environment -and $azContext.Environment.PSObject.Properties.Name -contains 'Name') {
            $environmentName = [string]$azContext.Environment.Name
        }
    }
    catch { Write-Debug "Invoke-AZSCGraphRequestParallel: Azure context not readable, assuming AzureCloud: $($_.Exception.Message)" }
    $baseUrl = switch ($environmentName) {
        'AzureUSGovernment' { 'https://graph.microsoft.us' }
        'AzureChinaCloud'   { 'https://microsoftgraph.chinacloudapi.cn' }
        default             { 'https://graph.microsoft.com' }
    }

    $index = 0
    $work = foreach ($target in $Uri) {
        [pscustomobject]@{
            Index   = $index++
            Uri     = $target
            FullUri = if ($target -match '^https?://') { $target } else { "$baseUrl$target" }
        }
    }

    $timer = [System.Diagnostics.Stopwatch]::StartNew()
    $firstPages = @{}
    try {
        $work | ForEach-Object -ThrottleLimit $ThrottleLimit -Parallel {
            $item = $_
            try {
                $response = Invoke-RestMethod -Uri $item.FullUri -Method GET -Headers @{ Authorization = $using:authorization } -TimeoutSec $using:TimeoutSec -ErrorAction Stop
                $hasValue = ($null -ne $response -and $response.PSObject.Properties.Name -contains 'value')
                $hasNext = ($null -ne $response -and $response.PSObject.Properties.Name -contains '@odata.nextLink')
                [pscustomobject]@{
                    Index    = $item.Index
                    Rows     = if ($hasValue) { @($response.value) } elseif ($null -ne $response) { @($response) } else { @() }
                    NextLink = if ($hasNext) { [string]$response.'@odata.nextLink' } else { '' }
                    Error    = ''
                }
            }
            catch {
                [pscustomobject]@{ Index = $item.Index; Rows = @(); NextLink = ''; Error = [string]$_.Exception.Message }
            }
        } | ForEach-Object { $firstPages[[int]$_.Index] = $_ }
    }
    catch {
        Write-Debug "Invoke-AZSCGraphRequestParallel: concurrent read unavailable, reading one at a time: $($_.Exception.Message)"
        $firstPages = @{}
    }
    Write-Debug ((Get-Date -Format 'yyyy-MM-dd_HH_mm_ss') + " - Graph concurrent first pages: $($firstPages.Count) of $(@($work).Count) in $($timer.ElapsedMilliseconds) ms")

    foreach ($item in $work) {
        $page = if ($firstPages.ContainsKey([int]$item.Index)) { $firstPages[[int]$item.Index] } else { $null }
        if ($null -eq $page -or -not [string]::IsNullOrEmpty($page.Error)) {
            if ($null -ne $page) {
                Write-Debug "Invoke-AZSCGraphRequestParallel: first page failed for $($item.FullUri), re-reading with retries: $($page.Error)"
            }
            & $serialRead $item.Uri
            continue
        }
        foreach ($row in @($page.Rows)) {
            if ($null -ne $row) { $rows.Add($row) }
        }
        if (-not [string]::IsNullOrEmpty($page.NextLink)) { & $serialRead $page.NextLink }
    }

    return $rows.ToArray()
}