Modules/MDOP2.Discovery.psm1
|
Set-StrictMode -Version Latest # Message trace discovery (delegated Exchange Online session) and mailbox lookup (app-only Graph). $script:TraceHistory = [Collections.Generic.List[datetime]]::new() $script:TraceResultSize = 5000 $script:RecipientCache = @{} $script:FolderCache = @{} $script:MessageSelect = 'id,subject,receivedDateTime,parentFolderId,internetMessageId' $script:MailboxDirectory = $null $script:MailboxDirectoryMax = 5000 $script:DirectoryWarmup = 20 # Trace statuses that mean the message never reached a mailbox $script:NoMailboxStatus = 'Failed', 'Quarantined', 'Expanded' function Invoke-MDOP2TraceQuery { param([hashtable]$Parameters, [int]$MaxAttempts = 3) for ($attempt = 1; ; $attempt++) { $wait = Get-MDOP2ThrottleDelay -History $script:TraceHistory.ToArray() if ($wait -gt 0) { Write-Host "Message trace limit of 100 queries per 5 minutes reached. Waiting $wait s." -ForegroundColor Yellow Start-Sleep -Seconds $wait } $now = [datetime]::UtcNow $script:TraceHistory.Add($now) $script:TraceHistory.RemoveAll({ param($t) ($now - $t).TotalSeconds -ge 300 }) | Out-Null try { return @(Get-MessageTraceV2 @Parameters -ErrorAction Stop) } catch { if ($attempt -lt $MaxAttempts -and $_.Exception.Message -match 'throttl|too many|429|temporar|timeout') { $delay = 30 * $attempt Write-Verbose "Message trace failed ($($_.Exception.Message)). Retrying in $delay s." Start-Sleep -Seconds $delay continue } throw } } } function Search-MDOP2MessageTrace { <# .SYNOPSIS Finds messages with Get-MessageTraceV2. Splits the range into 10 day windows, continues past 5000 rows per window, applies domain filters locally and stops at MaxRows. #> [CmdletBinding()] param( [Parameter(Mandatory)][datetime]$StartDate, [Parameter(Mandatory)][datetime]$EndDate, [string]$Subject, [ValidateSet('Contains', 'StartsWith', 'EndsWith')][string]$SubjectFilterType = 'Contains', [string[]]$Sender, [string[]]$Recipient, [ValidateSet('Delivered', 'Expanded', 'Failed', 'FilteredAsSpam', 'GettingStatus', 'Pending', 'Quarantined')][string[]]$Status, [string]$MessageId, [ValidateRange(1, 100000)][int]$MaxRows = 20000 ) $StartDate = ConvertTo-MDOP2UtcTimestamp $StartDate $EndDate = ConvertTo-MDOP2UtcTimestamp $EndDate $oldest = [datetime]::UtcNow.AddDays(-90) if ($StartDate -lt $oldest) { throw 'Message trace only covers the last 90 days.' } if ($Subject -and $Subject -match '[\x00-\x1F]') { throw 'Subject contains control characters.' } $senderFilter = ConvertTo-MDOP2AddressFilter -Value $Sender $recipientFilter = ConvertTo-MDOP2AddressFilter -Value $Recipient $base = @{ ResultSize = $script:TraceResultSize } if ($Subject) { $base.Subject = $Subject; $base.SubjectFilterType = $SubjectFilterType } if ($senderFilter.ServerSide) { $base.SenderAddress = $senderFilter.Addresses } if ($recipientFilter.ServerSide) { $base.RecipientAddress = $recipientFilter.Addresses } if ($Status) { $base.Status = $Status } if ($MessageId) { $base.MessageId = $MessageId } $rows = [Collections.Generic.List[object]]::new() $seen = [Collections.Generic.HashSet[string]]::new() $queries = 0 $truncated = $false foreach ($w in Split-MDOP2DateRange -Start $StartDate -End $EndDate) { $end = $w.End $startingRecipient = $null while ($true) { $p = $base.Clone() # Pass local times: the cmdlet reads dates in the local time zone $p.StartDate = $w.Start.ToLocalTime() $p.EndDate = $end.ToLocalTime() if ($startingRecipient) { $p.StartingRecipientAddress = $startingRecipient } # @() because an empty window returns nothing, and $null.Count throws under StrictMode $batch = @(Invoke-MDOP2TraceQuery -Parameters $p) $queries++ Write-Verbose "Trace window $($w.Start.ToString('u')) to $($end.ToString('u')): $($batch.Count) rows." $newRows = 0 foreach ($r in $batch) { $key = "$($r.MessageTraceId)|$($r.RecipientAddress)" if (-not $seen.Add($key)) { continue } $newRows++ if (-not (Test-MDOP2AddressMatch -Address $r.SenderAddress -Filter $senderFilter)) { continue } if (-not (Test-MDOP2AddressMatch -Address $r.RecipientAddress -Filter $recipientFilter)) { continue } $rows.Add([pscustomobject]@{ Received = ConvertTo-MDOP2UtcTimestamp $r.Received SenderAddress = [string]$r.SenderAddress RecipientAddress = [string]$r.RecipientAddress Subject = [string]$r.Subject Status = [string]$r.Status MessageId = [string]$r.MessageId MessageTraceId = [string]$r.MessageTraceId FromIP = [string]$r.FromIP Size = $r.Size }) if ($rows.Count -ge $MaxRows) { $truncated = $true; break } } # A full batch with nothing new means the continuation isn't moving; stop instead of looping if ($truncated -or $batch.Count -lt $script:TraceResultSize -or $newRows -eq 0) { break } $last = $batch[$batch.Count - 1] $end = ConvertTo-MDOP2UtcTimestamp $last.Received $startingRecipient = [string]$last.RecipientAddress } if ($truncated) { break } } $groups = @(Group-MDOP2TraceResult -Rows $rows.ToArray()) Write-MDOP2Audit -Type Search -Data ([ordered]@{ filters = [ordered]@{ startDate = $StartDate.ToString('o'); endDate = $EndDate.ToString('o') subject = $Subject; subjectFilterType = $(if ($Subject) { $SubjectFilterType }) sender = @($Sender | Where-Object { $_ }); recipient = @($Recipient | Where-Object { $_ }) status = @($Status | Where-Object { $_ }); messageId = $MessageId } rows = $rows.Count messages = $groups.Count queries = $queries truncated = $truncated }) [pscustomobject]@{ Rows = $rows.ToArray() Groups = $groups Queries = $queries Truncated = $truncated } } function Get-MDOP2Recipient { <# .SYNOPSIS Looks up an address in Exchange Online (cached). Returns the Entra object ID and recipient type, or nothing for unknown addresses. #> [CmdletBinding()] param([Parameter(Mandatory)][string]$Address) $key = $Address.ToLowerInvariant() if ($script:RecipientCache.ContainsKey($key)) { return $script:RecipientCache[$key] } $result = $null try { $r = Get-EXORecipient -Identity $Address -Properties PrimarySmtpAddress -ErrorAction Stop if ($r) { $result = [pscustomobject]@{ ObjectId = [string]$r.ExternalDirectoryObjectId RecipientTypeDetails = [string]$r.RecipientTypeDetails PrimarySmtpAddress = [string]$r.PrimarySmtpAddress } } } catch { if ($_.Exception.Message -notmatch "couldn't be found|not found|ManagementObjectNotFound") { throw } } $script:RecipientCache[$key] = $result $result } function Get-MDOP2MailboxDirectory { <# .SYNOPSIS Lists the user and shared mailboxes of the tenant for the recipient picker, once per session (at most MailboxDirectoryMax). Every SMTP address of these mailboxes also goes into the recipient cache, so locating mail needs no Get-EXORecipient call for them. #> [CmdletBinding()] param() if ($script:MailboxDirectory) { return $script:MailboxDirectory } $list = @(Get-EXOMailbox -ResultSize ($script:MailboxDirectoryMax + 1) -RecipientTypeDetails UserMailbox, SharedMailbox -ErrorAction Stop) $truncated = $list.Count -gt $script:MailboxDirectoryMax if ($truncated) { $list = $list[0..($script:MailboxDirectoryMax - 1)] } $items = foreach ($m in $list) { $info = [pscustomobject]@{ ObjectId = [string]$m.ExternalDirectoryObjectId RecipientTypeDetails = [string]$m.RecipientTypeDetails PrimarySmtpAddress = [string]$m.PrimarySmtpAddress } foreach ($a in @($m.EmailAddresses)) { if ("$a" -match '^smtp:(.+)$') { $script:RecipientCache[$Matches[1].ToLowerInvariant()] = $info } } [pscustomobject]@{ DisplayName = [string]$m.DisplayName Address = [string]$m.PrimarySmtpAddress Upn = [string]$m.UserPrincipalName Type = [string]$m.RecipientTypeDetails } } $script:MailboxDirectory = [pscustomobject]@{ Mailboxes = @($items | Sort-Object DisplayName, Address); Truncated = $truncated } $script:MailboxDirectory } function Initialize-MDOP2RecipientCache { # Many unknown addresses: one mailbox list is quicker than a Get-EXORecipient call per address param([AllowEmptyCollection()][object[]]$Group) if ($script:MailboxDirectory) { return } $unknown = [Collections.Generic.HashSet[string]]::new() foreach ($g in $Group) { foreach ($rc in $g.Recipients) { if ($rc.Status -in $script:NoMailboxStatus) { continue } $key = "$($rc.Address)".ToLowerInvariant() if (-not $script:RecipientCache.ContainsKey($key)) { $null = $unknown.Add($key) } } } if ($unknown.Count -gt $script:DirectoryWarmup) { try { $null = Get-MDOP2MailboxDirectory } catch { Write-Verbose "Mailbox list failed: $($_.Exception.Message)" } } } function Find-MDOP2MessageCopy { <# .SYNOPSIS Finds every copy of one or more messages by Message-ID with app-only Graph JSON batching, including Recoverable Items. Takes lookups with UserId and InternetMessageId and returns one result per lookup, in the same order. #> [CmdletBinding()] param([Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Lookup) $list = @($Lookup) if (-not $list.Count) { return } $requests = [Collections.Generic.List[object]]::new() for ($i = 0; $i -lt $list.Count; $i++) { $uid = [uri]::EscapeDataString([string]$list[$i].UserId) $filter = [uri]::EscapeDataString("internetMessageId eq $(ConvertTo-MDOP2ODataLiteral ([string]$list[$i].InternetMessageId))") $query = "?`$filter=$filter&`$select=$script:MessageSelect" $requests.Add(@{ Id = "m$i"; Url = "/users/$uid/messages$query" }) $requests.Add(@{ Id = "r$i"; Url = "/users/$uid/mailFolders/recoverableitemsdeletions/messages$query" }) } $res = Invoke-MDOP2AppGraphBatch -Request $requests.ToArray() -All $errors = [object[]]::new($list.Count) $copies = [object[]]::new($list.Count) $folders = [ordered]@{} for ($i = 0; $i -lt $list.Count; $i++) { $uid = [uri]::EscapeDataString([string]$list[$i].UserId) $inMailbox = $res["m$i"] $inRecoverable = $res["r$i"] if ($inMailbox.Error) { $errors[$i] = $inMailbox; continue } if ($inRecoverable.Error) { $errors[$i] = $inRecoverable; continue } $messages = [ordered]@{} foreach ($m in @($inMailbox.Value) + @($inRecoverable.Value)) { if (-not $messages.Contains($m.id)) { $messages[$m.id] = $m } } $copies[$i] = $messages foreach ($m in $messages.Values) { $key = "$uid|$($m.parentFolderId)" if (-not $script:FolderCache.ContainsKey($key) -and -not $folders.Contains($key)) { $folders[$key] = @{ Id = "f$($folders.Count)"; Url = "/users/$uid/mailFolders/$([uri]::EscapeDataString([string]$m.parentFolderId))?`$select=displayName" } } } } if ($folders.Count) { $names = Invoke-MDOP2AppGraphBatch -Request @($folders.Values) foreach ($key in $folders.get_Keys()) { $n = $names[$folders[$key].Id] # Only names that were read are kept, so a failed lookup is tried again next time if ($n -and -not $n.Error -and $n.Body -and $n.Body.PSObject.Properties['displayName']) { $script:FolderCache[$key] = [string]$n.Body.displayName } } } for ($i = 0; $i -lt $list.Count; $i++) { if ($errors[$i]) { [pscustomobject]@{ State = Get-MDOP2MailboxErrorState -StatusCode $errors[$i].Status -ErrorCode $errors[$i].ErrorCode Locations = @() Error = $errors[$i].Error } continue } $uid = [uri]::EscapeDataString([string]$list[$i].UserId) $locations = foreach ($m in $copies[$i].Values) { [pscustomobject]@{ GraphMessageId = [string]$m.id FolderId = [string]$m.parentFolderId FolderName = $script:FolderCache["$uid|$($m.parentFolderId)"] ?? '(unknown folder)' ReceivedDateTime = $m.receivedDateTime Subject = [string]$m.subject } } [pscustomobject]@{ State = if ($copies[$i].Count) { 'Found' } else { 'NotFound' } Locations = @($locations) Error = $null } } } function Find-MDOP2MailboxMessage { <# .SYNOPSIS Finds every copy of a message in one mailbox by Message-ID with app-only Graph, including Recoverable Items. #> [CmdletBinding()] param( [Parameter(Mandatory)][string]$UserId, [Parameter(Mandatory)][string]$InternetMessageId ) Find-MDOP2MessageCopy -Lookup @(@{ UserId = $UserId; InternetMessageId = $InternetMessageId }) } function Resolve-MDOP2MessageGroup { <# .SYNOPSIS For each recipient of one or more grouped messages, finds where the message is now. Recipients that can't hold it are skipped with a reason. The mailbox lookups of all groups run together through Graph JSON batching. Each row has the GroupKey. #> [CmdletBinding()] param([Parameter(Mandatory)][object[]]$Group) Initialize-MDOP2RecipientCache -Group $Group $rows = [Collections.Generic.List[object]]::new() $lookups = [Collections.Generic.List[object]]::new() $targets = [Collections.Generic.List[object]]::new() # An alias and the primary address of one mailbox can both be trace recipients; look that mailbox up once $lookupIndex = [Collections.Generic.Dictionary[string, int]]::new([StringComparer]::Ordinal) $total = 0 foreach ($g in $Group) { $total += @($g.Recipients).Count } $i = 0 foreach ($g in $Group) { foreach ($rc in $g.Recipients) { $i++ Write-Progress -Activity 'Looking up recipients' -Status $rc.Address -PercentComplete (100 * $i / [Math]::Max($total, 1)) $recipient = $null if ($rc.Status -notin $script:NoMailboxStatus) { $recipient = Get-MDOP2Recipient -Address $rc.Address } $type = if ($recipient) { $recipient.RecipientTypeDetails } else { $null } $skip = Get-MDOP2ResolveSkipReason -TraceStatus $rc.Status -RecipientTypeDetails $type if (-not $skip -and -not $g.InternetMessageId) { $skip = 'Message trace has no Message-ID for this message' } $row = [pscustomobject][ordered]@{ GroupKey = $g.Key Address = $rc.Address TraceStatus = $rc.Status RecipientTypeDetails = $type UserId = if ($recipient) { $recipient.ObjectId } else { $null } State = 'Skipped' Locations = @() Detail = $skip } $rows.Add($row) if (-not $skip) { $lk = "$($recipient.ObjectId)|$($g.InternetMessageId)" if (-not $lookupIndex.ContainsKey($lk)) { $lookupIndex[$lk] = $lookups.Count $lookups.Add(@{ UserId = $recipient.ObjectId; InternetMessageId = $g.InternetMessageId }) } $targets.Add(@{ Row = $row; Index = $lookupIndex[$lk] }) } } } Write-Progress -Activity 'Looking up recipients' -Completed if ($lookups.Count) { Write-Progress -Activity 'Locating messages in mailboxes' -Status "$($lookups.Count) mailbox copies" $found = @(Find-MDOP2MessageCopy -Lookup $lookups.ToArray()) foreach ($t in $targets) { $f = $found[$t.Index] $t.Row.State = $f.State $t.Row.Locations = $f.Locations $t.Row.Detail = $f.Error } Write-Progress -Activity 'Locating messages in mailboxes' -Completed } $rows.ToArray() } function Get-MDOP2GroupBody { <# .SYNOPSIS Reads the body of one copy of each grouped message for the content filter, with Graph JSON batching. Tries up to MaxMailboxes recipients per message and returns a hashtable of group key to body; messages without a readable copy are left out. Bodies stay in memory and are never logged. #> [CmdletBinding()] param([Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Group, [int]$MaxMailboxes = 3) Initialize-MDOP2RecipientCache -Group $Group $bodies = @{} # Each round asks one place per message: a mailbox, then its Recoverable Items, then the next mailbox. # Most messages are found in the first place, so this sends far fewer requests than asking every place at once. $todo = [Collections.Generic.List[hashtable]]::new() foreach ($g in $Group) { if ($g.InternetMessageId) { $todo.Add(@{ Group = $g; Next = 0; Tried = 0; UserId = $null; Recoverable = $false }) } } while ($todo.Count) { $requests = [Collections.Generic.List[object]]::new() $sent = @{} foreach ($t in $todo.ToArray()) { if (-not $t.Recoverable) { $t.UserId = $null $list = @($t.Group.Recipients) while (-not $t.UserId -and $t.Tried -lt $MaxMailboxes -and $t.Next -lt $list.Count) { $rc = $list[$t.Next] $t.Next++ if ($rc.Status -in $script:NoMailboxStatus) { continue } $recipient = Get-MDOP2Recipient -Address $rc.Address if (-not (Get-MDOP2ResolveSkipReason -TraceStatus $rc.Status -RecipientTypeDetails $(if ($recipient) { $recipient.RecipientTypeDetails }))) { $t.UserId = $recipient.ObjectId $t.Tried++ } } if (-not $t.UserId) { $null = $todo.Remove($t); continue } } $filter = [uri]::EscapeDataString("internetMessageId eq $(ConvertTo-MDOP2ODataLiteral $t.Group.InternetMessageId)") $folder = if ($t.Recoverable) { '/mailFolders/recoverableitemsdeletions' } else { '' } $id = "b$($requests.Count)" $sent[$id] = $t $requests.Add(@{ Id = $id; Url = "/users/$([uri]::EscapeDataString($t.UserId))$folder/messages?`$filter=$filter&`$select=body&`$top=1" }) } if (-not $requests.Count) { break } $res = Invoke-MDOP2AppGraphBatch -Request $requests.ToArray() foreach ($id in $sent.get_Keys()) { $t = $sent[$id] $r = $res[$id] $m = $null if ($r -and -not $r.Error -and $r.Body -and $r.Body.PSObject.Properties['value']) { $m = @($r.Body.value) | Select-Object -First 1 } if ($m -and $m.PSObject.Properties['body'] -and $m.body) { $bodies[$t.Group.Key] = [string]$m.body.content $null = $todo.Remove($t) } elseif (-not $r -or $r.Error) { # The mailbox can't be read; its Recoverable Items can't either, so go to the next mailbox if ($r) { Write-Verbose "Body lookup failed: $($r.Error)" } $t.Recoverable = $false } else { $t.Recoverable = -not $t.Recoverable } } $res = $null } $bodies } Export-ModuleMember -Function Search-MDOP2MessageTrace, Get-MDOP2Recipient, Get-MDOP2MailboxDirectory, Find-MDOP2MessageCopy, Find-MDOP2MailboxMessage, Resolve-MDOP2MessageGroup, Get-MDOP2GroupBody |