Private/Invoke-RaidinessGraphCollect.ps1
|
<# The Microsoft Graph collection step of Invoke-Raidiness, on its own so the orchestrator above it stays readable. Walks collect.manifest.json and issues only GET requests; see Invoke-Raidiness for the evidence shapes. Progress uses three nested bars: the caller owns id 1 (the phases of the run), this function owns id 2 (collector n of 52) and id 3 (the evidence item, and the page inside a paged one). Without id 3 a tenant with 40 000 sign-ins looks exactly like a hang. Skips are objects rather than sentences now. They used to be strings that Invoke-Raidiness reduced to a count, which threw away the only answer to "why is this not measured?" -- they are returned, logged, and written to the run folder. #> # psrunner-lint allow: New-Item — creates the local output directory for evidence files; never a tenant object # psrunner-lint allow: Remove-Item — deletes the local temp file a CSV report was downloaded to; never a tenant object function Invoke-RaidinessGraphCollect { [CmdletBinding()] param( [string] $OutputPath = './raidiness-data', [switch] $IncludeSlow, [string[]] $Collector, [switch] $Quiet, # Microsoft Graph SDK's per-request client timeout. This is separate # from the browser/report timeout exposed by the checkup command. [ValidateRange(1, 2147483647)] [int] $GraphRequestTimeoutSeconds = 100, # A runaway @odata.nextLink chain is the most likely way a collection # hangs. Stop, record it, and let the checks read a truncated source # as what it is rather than following the link forever. [int] $MaxPage = 200 ) $ErrorActionPreference = 'Stop' if (-not (Get-MgContext)) { throw 'Not connected to Microsoft Graph. Run Connect-Raidiness first.' } $manifestPath = Join-Path $PSScriptRoot '..' 'collect.manifest.json' $manifest = Get-Content -Path $manifestPath -Raw | ConvertFrom-Json $grantedScopes = @((Get-MgContext).Scopes) New-Item -ItemType Directory -Path $OutputPath -Force | Out-Null $collected = 0 $skipped = [System.Collections.Generic.List[object]]::new() $notes = [System.Collections.Generic.List[string]]::new() $activity = 'Raidiness: collecting from Microsoft Graph' $entries = @($manifest.collectors | Where-Object { -not $Collector -or $_.key -in $Collector }) $total = [math]::Max($entries.Count, 1) $position = 0 $started = Get-Date # ClientTimeout is process-wide SDK context, rather than an argument on # Invoke-MgGraphRequest. Only change it when this SDK version exposes the # request-context commands, and put the caller's value back below. $restoreRequestTimeout = $false $previousRequestTimeout = $null $getRequestContext = Get-Command -Name Get-MgRequestContext -ErrorAction SilentlyContinue $setRequestContext = Get-Command -Name Set-MgRequestContext -ErrorAction SilentlyContinue if ($getRequestContext -and $setRequestContext -and $setRequestContext.Parameters.ContainsKey('ClientTimeout')) { $requestContext = Get-MgRequestContext if ($requestContext -and $requestContext.PSObject.Properties['ClientTimeout']) { $previousRequestTimeout = $requestContext.ClientTimeout Set-MgRequestContext -ClientTimeout $GraphRequestTimeoutSeconds | Out-Null $restoreRequestTimeout = $true } } Write-RaidinessLog -Phase 'collect' -Message "$($entries.Count) collector(s) selected" try { foreach ($entry in $entries) { $position++ $elapsed = ((Get-Date) - $started).TotalSeconds $remaining = $position -gt 1 ? [int]($elapsed / ($position - 1) * ($total - $position + 1)) : -1 Write-RaidinessProgress -Id 2 -ParentId 1 -Quiet:$Quiet -Activity $activity ` -Status "$($entry.key) ($position of $total)" ` -PercentComplete ([int](100 * ($position - 1) / $total)) ` -SecondsRemaining $remaining $requiredScopes = @($entry.evidence.permission | Where-Object { $_ } | Sort-Object -Unique) $missingScopes = @($requiredScopes | Where-Object { $_ -notin $grantedScopes }) if ($missingScopes.Count -gt 0) { $groups = @($entry.optionalScopeGroups) -join ', ' $reason = "missing delegated scope(s): $($missingScopes -join ', ')" if ($groups) { $reason += "; reconnect with Connect-Raidiness -OptionalScopeGroup $groups" } $skipped.Add([pscustomobject]@{ Key = $entry.key; Kind = 'scope'; Endpoint = $null; Reason = $reason }) Write-RaidinessLog -Phase 'collect' -Source $entry.key -Message "skipped: $reason" continue } if ((Get-Member -InputObject $entry -Name 'slow') -and $entry.slow -and -not $IncludeSlow) { # Silent before: the operator could not tell a slow-skip from a # collector that was never in the manifest. $skipped.Add([pscustomobject]@{ Key = $entry.key; Kind = 'slow'; Endpoint = $null; Reason = 'marked slow in the manifest; re-run with -IncludeSlow' }) Write-RaidinessLog -Phase 'collect' -Source $entry.key -Message 'skipped: marked slow (use -IncludeSlow)' continue } $items = @() $failed = $null $failedKind = 'error' $evidenceCount = @($entry.evidence).Count $evidenceIndex = 0 $collectorStarted = Get-Date foreach ($evidence in $entry.evidence) { $evidenceIndex++ Write-RaidinessProgress -Id 3 -ParentId 2 -Quiet:$Quiet -Activity $entry.key ` -Status $evidence.endpoint ` -PercentComplete ([int](100 * ($evidenceIndex - 1) / [math]::Max($evidenceCount, 1))) try { $headers = @{} if ($evidence.endpoint -match '\$count|\$search') { $headers.ConsistencyLevel = 'eventual' } $item = [ordered]@{ endpoint = $evidence.endpoint method = $evidence.method capturedAt = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ') } if (Get-Member -InputObject $evidence -Name 'permission') { $item.permission = $evidence.permission } switch ($evidence.kind) { 'paged' { $rows = [System.Collections.Generic.List[object]]::new() $next = $evidence.endpoint $page = 0 while ($next) { $page++ if ($page -gt $MaxPage) { $note = "$($entry.key): stopped after $MaxPage pages ($($rows.Count) rows); the source is truncated" $notes.Add($note) $skipped.Add([pscustomobject]@{ Key = $entry.key; Kind = 'truncated'; Endpoint = $evidence.endpoint; Reason = "more than $MaxPage pages" }) Write-RaidinessLog -Level warn -Phase 'collect' -Source $entry.key -Message $note break } Write-RaidinessProgress -Id 3 -ParentId 2 -Quiet:$Quiet -Activity $entry.key ` -Status "$($evidence.endpoint) — page $page, $($rows.Count) rows" $response = Invoke-MgGraphRequest -Method GET -Uri $next -Headers $headers -OutputType PSObject if ($null -ne $response.PSObject.Properties['value']) { $rows.AddRange(@($response.value)) } $next = $response.PSObject.Properties['@odata.nextLink'] ? $response.'@odata.nextLink' : $null } $item.payload = $rows.ToArray() } 'single' { $item.payload = Invoke-MgGraphRequest -Method GET -Uri $evidence.endpoint -Headers $headers -OutputType PSObject } 'csv' { Write-RaidinessProgress -Id 3 -ParentId 2 -Quiet:$Quiet -Activity $entry.key -Status "$($evidence.endpoint) — downloading report CSV" $temporary = Join-Path ([System.IO.Path]::GetTempPath()) ("raidiness-" + [guid]::NewGuid() + '.csv') try { Invoke-MgGraphRequest -Method GET -Uri $evidence.endpoint -OutputFilePath $temporary $rows = @(Import-Csv -Path $temporary) $item.payload = $rows if ($rows.Count -gt 0 -and $rows[0].PSObject.Properties['Report Refresh Date']) { $item.reportRefreshDate = $rows[0].'Report Refresh Date' } } finally { Remove-Item -Path $temporary -ErrorAction SilentlyContinue } } } $items += [pscustomobject]$item } catch { $exception = $_.Exception $isTimeout = $false while ($exception) { if ($exception -is [System.TimeoutException] -or $exception -is [System.Threading.Tasks.TaskCanceledException] -or $exception.Message -match '(?i)timed?\s*out|timeout|operation (was )?cancell?ed') { $isTimeout = $true break } $exception = $exception.InnerException } if ($isTimeout) { $failedKind = 'timeout' $failed = "request timed out after $GraphRequestTimeoutSeconds seconds" } else { $failed = $_.Exception.Message } break } } if ($failed) { # A partially collected file could mislead measure(); leave the # whole collector out and let its checks degrade to "not measured". $skip = [ordered]@{ Key = $entry.key; Kind = $failedKind; Endpoint = $evidence.endpoint; Reason = $failed } if ($failedKind -eq 'timeout') { $skip.TimeoutSeconds = $GraphRequestTimeoutSeconds } $skipped.Add([pscustomobject]$skip) Write-RaidinessLog -Level warn -Phase 'collect' -Source $entry.key -Message "skipped: $failed" -Data @{ endpoint = $evidence.endpoint } Write-Warning "Skipped $($entry.key): $failed" continue } $file = Join-Path $OutputPath "$($entry.key).json" ConvertTo-Json -InputObject $items -Depth 48 -Compress | Out-File -FilePath $file -Encoding utf8 $collected++ $rowCount = @($items | ForEach-Object { @($_.payload).Count } | Measure-Object -Sum).Sum Write-RaidinessLog -Phase 'collect' -Source $entry.key -Message "$rowCount row(s) from $evidenceCount endpoint(s)" -DurationMs ((Get-Date) - $collectorStarted).TotalMilliseconds if (-not $Quiet) { Write-Host " collected $($entry.key)" -ForegroundColor DarkGray } # The bar reaches 100 only if it is also written after the work; it # used to report ($position - 1) and stop one collector short. Write-RaidinessProgress -Id 2 -ParentId 1 -Quiet:$Quiet -Activity $activity -Status $entry.key -PercentComplete ([int](100 * $position / $total)) } } finally { if ($restoreRequestTimeout) { Set-MgRequestContext -ClientTimeout $previousRequestTimeout | Out-Null } Write-RaidinessProgress -Id 3 -Quiet:$Quiet -Activity 'done' -Completed Write-RaidinessProgress -Id 2 -Quiet:$Quiet -Activity $activity -Completed } [pscustomobject]@{ Collected = $collected; Skipped = $skipped.ToArray(); Notes = $notes.ToArray() } } |