Public/Get-NLBaselineCAReportOnlyAnalysis.ps1
|
function Get-NLBaselineCAReportOnlyAnalysis { <# .SYNOPSIS Analyze Report-Only Conditional Access policy results from sign-in logs .DESCRIPTION Analyzes sign-in logs to identify which policies in report-only mode would have been applied. Provides insights into policy impact before enabling them. .EXAMPLE Get-NLBaselineCAReportOnlyAnalysis #> [CmdletBinding()] param( [Parameter(Mandatory = $false)] [ValidateRange(1, 168)] [int]$Hours = 24 ) try { # Check connection $context = Get-MgContext -ErrorAction SilentlyContinue if (-not $context -or -not $context.TenantId) { Write-Host "Not connected to Microsoft 365. Connecting..." -ForegroundColor Yellow Write-Host "" $connection = Connect-NLBaselineCA if (-not $connection) { Write-Error "Cannot connect to Microsoft 365" return } $context = Get-MgContext } # Get module configuration $moduleConfigPath = Get-ConfigPath if (-not (Test-Path $moduleConfigPath)) { Write-Error "Module configuration not found. Run Quick Start first." return } $moduleConfig = Get-Content $moduleConfigPath | ConvertFrom-Json $storagePath = $moduleConfig.StoragePath if (-not (Test-Path $storagePath)) { Write-Error "Storage path not found: $storagePath. Run Quick Start to configure." return } Write-Host "========================================" -ForegroundColor Cyan Write-Host " REPORT-ONLY POLICY ANALYSIS" -ForegroundColor Cyan Write-Host "========================================" -ForegroundColor Cyan Write-Host "" Write-Host "Analyzing sign-in logs for the past $Hours hours..." -ForegroundColor Yellow Write-Host "" # Calculate date range $startDate = (Get-Date).AddHours(-$Hours) # Get sign-in logs Write-Host "Fetching sign-in logs..." -ForegroundColor Gray $signIns = Get-MgAuditLogSignIn -Filter "createdDateTime ge $($startDate.ToString('yyyy-MM-ddTHH:mm:ssZ'))" -All -ErrorAction Stop Write-Host "Found $($signIns.Count) sign-in records" -ForegroundColor Green Write-Host "" # Analyze report-only policies $reportOnlyResults = @() $policyStats = @{} $userStats = @{} foreach ($signIn in $signIns) { $reportOnlyPolicies = $signIn.AppliedConditionalAccessPolicies | Where-Object { $_.Result -like 'reportOnly*' } if ($reportOnlyPolicies) { foreach ($policy in $reportOnlyPolicies) { # Track policy statistics if (-not $policyStats.ContainsKey($policy.Id)) { $policyStats[$policy.Id] = @{ PolicyName = $policy.DisplayName PolicyId = $policy.Id Count = 0 Users = @{} Apps = @{} GrantControls = @{} } } $policyStats[$policy.Id].Count++ # Track unique users if ($signIn.UserPrincipalName) { if (-not $policyStats[$policy.Id].Users.ContainsKey($signIn.UserPrincipalName)) { $policyStats[$policy.Id].Users[$signIn.UserPrincipalName] = 0 } $policyStats[$policy.Id].Users[$signIn.UserPrincipalName]++ } # Track apps if ($signIn.AppDisplayName) { if (-not $policyStats[$policy.Id].Apps.ContainsKey($signIn.AppDisplayName)) { $policyStats[$policy.Id].Apps[$signIn.AppDisplayName] = 0 } $policyStats[$policy.Id].Apps[$signIn.AppDisplayName]++ } # Track grant controls foreach ($control in $policy.EnforcedGrantControls) { if (-not $policyStats[$policy.Id].GrantControls.ContainsKey($control)) { $policyStats[$policy.Id].GrantControls[$control] = 0 } $policyStats[$policy.Id].GrantControls[$control]++ } # Add to detailed results $reportOnlyResults += [PSCustomObject]@{ Time = $signIn.CreatedDateTime UserPrincipalName = $signIn.UserPrincipalName UserDisplayName = $signIn.UserDisplayName AppDisplayName = $signIn.AppDisplayName ClientApp = $signIn.ClientAppUsed DevicePlatform = $signIn.DevicePlatform Location = if ($signIn.Location.City) { "$($signIn.Location.City), $($signIn.Location.CountryOrRegion)" } else { $signIn.Location.CountryOrRegion } RiskLevel = $signIn.RiskLevelAggregated PolicyId = $policy.Id PolicyName = $policy.DisplayName Result = $policy.Result GrantControls = ($policy.EnforcedGrantControls -join '; ') SessionControls = ($policy.EnforcedSessionControls -join '; ') } } } } # Display results if ($reportOnlyResults.Count -eq 0) { Write-Host "No report-only policy matches found in the analyzed time period." -ForegroundColor Yellow Write-Host "" return } Write-Host "========================================" -ForegroundColor Green Write-Host " ANALYSIS RESULTS" -ForegroundColor Green Write-Host "========================================" -ForegroundColor Green Write-Host "" Write-Host "Total report-only policy matches: $($reportOnlyResults.Count)" -ForegroundColor White Write-Host "Unique policies triggered: $($policyStats.Count)" -ForegroundColor White Write-Host "" # Display policy statistics Write-Host "========================================" -ForegroundColor Cyan Write-Host " POLICY STATISTICS" -ForegroundColor Cyan Write-Host "========================================" -ForegroundColor Cyan Write-Host "" $sortedPolicies = $policyStats.Values | Sort-Object Count -Descending foreach ($policyStat in $sortedPolicies) { Write-Host "Policy: $($policyStat.PolicyName)" -ForegroundColor Yellow Write-Host " Matches: $($policyStat.Count)" -ForegroundColor White Write-Host " Unique users affected: $($policyStat.Users.Count)" -ForegroundColor Gray Write-Host " Unique apps: $($policyStat.Apps.Count)" -ForegroundColor Gray Write-Host " Grant controls that would apply:" -ForegroundColor Gray foreach ($control in $policyStat.GrantControls.Keys) { Write-Host " - $control : $($policyStat.GrantControls[$control]) times" -ForegroundColor DarkGray } Write-Host "" } # Save detailed report $reportPath = Join-Path $storagePath "ReportOnlyAnalysis" if (-not (Test-Path $reportPath)) { New-Item -Path $reportPath -ItemType Directory -Force | Out-Null } $timestamp = Get-Date -Format "yyyyMMdd-HHmmss" $reportFile = Join-Path $reportPath "report-only-analysis-$timestamp.json" $reportFileCsv = Join-Path $reportPath "report-only-analysis-$timestamp.csv" # Save JSON report $report = @{ AnalysisDate = (Get-Date).ToString("o") HoursAnalyzed = $Hours StartDate = $startDate.ToString("o") EndDate = (Get-Date).ToString("o") TotalSignIns = $signIns.Count TotalReportOnlyMatches = $reportOnlyResults.Count UniquePolicies = $policyStats.Count PolicyStatistics = $policyStats.Values DetailedResults = $reportOnlyResults } $report | ConvertTo-Json -Depth 10 | Out-File -FilePath $reportFile -Encoding UTF8 # Save CSV report $reportOnlyResults | Export-Csv -Path $reportFileCsv -NoTypeInformation -Encoding UTF8 Write-Host "========================================" -ForegroundColor Green Write-Host " REPORTS SAVED" -ForegroundColor Green Write-Host "========================================" -ForegroundColor Green Write-Host "JSON report: $reportFile" -ForegroundColor White Write-Host "CSV report: $reportFileCsv" -ForegroundColor White Write-Host "" } catch { Write-Error "Error analyzing report-only policies: $_" } } |