Modules/MDOP2.Jobs.psm1
|
Set-StrictMode -Version Latest # Runs web UI jobs on the main thread, where the sign-in, the session secret and the Exchange Online connection live. $script:Groups = @{} $script:Resolved = @{} $script:Plans = @{} $script:History = $null $script:ContentCheckMax = 300 # Similar emails: rows per trace, messages shown, and the shortest subject worth a trace of its own $script:SimilarTraceRows = 5000 $script:SimilarMax = 1000 $script:SimilarSubjectMin = 4 $script:FolderKinds = [ordered]@{ inbox = 'Inbox' archive = 'Archive' junkemail = 'Junk' deleteditems = 'Deleted' recoverableitemsdeletions = 'Recoverable' } function Set-MDOP2JobProgress { param([hashtable]$Job, [string]$Text) if ($Job) { $Job.Progress = $Text } } function Get-MDOP2Group { param([string]$Key) $g = $script:Groups[$Key] if (-not $g) { throw 'That message is no longer in the result list. Search again.' } $g } function Get-MDOP2FolderKind { <# .SYNOPSIS Classifies a folder by comparing its ID with the mailbox's well-known folders (display names are localised). Map comes from Get-MDOP2WellKnownFolderMap. #> param([hashtable]$Map, [string]$UserId, [string]$FolderId) if ($UserId -and $FolderId -and $Map.ContainsKey($UserId)) { $ids = $Map[$UserId] foreach ($name in $script:FolderKinds.Keys) { if ($ids[$name] -eq $FolderId) { return $script:FolderKinds[$name] } } } 'Other' } function Get-MDOP2LocationView { param([hashtable]$Map, [string]$UserId, [AllowNull()][AllowEmptyCollection()][object[]]$Locations) , @(foreach ($l in $Locations) { if ($l) { [ordered]@{ folder = $l.FolderName; kind = Get-MDOP2FolderKind -Map $Map -UserId $UserId -FolderId $l.FolderId } } }) } function Get-MDOP2FolderMapFor { # Well-known folder IDs for the mailboxes that hold a copy, so each location can be classified param([AllowEmptyCollection()][object[]]$Row) $users = @(foreach ($r in $Row) { if ($r -and $r.State -eq 'Found' -and $r.UserId) { $r.UserId } }) if (-not $users.Count) { return @{} } Get-MDOP2WellKnownFolderMap -UserId $users -Name @($script:FolderKinds.Keys) } function Update-MDOP2Resolved { <# .SYNOPSIS Locates every copy of the given messages in one batched lookup and caches the rows per message key. #> param([string[]]$Key) $groups = @($Key | Select-Object -Unique | ForEach-Object { Get-MDOP2Group -Key $_ } | Where-Object { $_.InternetMessageId }) if (-not $groups.Count) { return } $byKey = @{} foreach ($g in $groups) { $byKey[$g.Key] = [Collections.Generic.List[object]]::new() } foreach ($row in Resolve-MDOP2MessageGroup -Group $groups) { $byKey[$row.GroupKey].Add($row) } foreach ($k in $byKey.get_Keys()) { $script:Resolved[$k] = $byKey[$k].ToArray() } } function Invoke-MDOP2SearchJob { param([object]$Payload, [hashtable]$Job) $range = Get-MDOP2DateRange -Range $Payload.Range $p = @{ StartDate = $range.Start; EndDate = $range.End } foreach ($k in $Payload.Parameters.Keys) { $p[$k] = $Payload.Parameters[$k] } Set-MDOP2JobProgress $Job 'Running message trace' $result = Search-MDOP2MessageTrace @p foreach ($g in $result.Groups) { $script:Groups[$g.Key] = $g } $groups = @($result.Groups) $content = $null if ($Payload.PSObject.Properties['Content'] -and $Payload.Content) { if ($groups.Count -gt $script:ContentCheckMax) { throw "Message trace found $($groups.Count) messages. The content filter reads each message, so narrow the search to $script:ContentCheckMax messages or fewer first." } Set-MDOP2JobProgress $Job "Checking the content of $($groups.Count) message(s)" $bodies = Get-MDOP2GroupBody -Group $groups $matched = @($groups | Where-Object { $bodies.ContainsKey($_.Key) -and (Test-MDOP2BodyMatch -Body $bodies[$_.Key] -Text $Payload.Content -Mode $Payload.ContentMode) }) $unchecked = @($groups | Where-Object { -not $bodies.ContainsKey($_.Key) }).Count $bodies = $null $content = [ordered]@{ mode = $Payload.ContentMode; checked = $groups.Count - $unchecked; unchecked = $unchecked; matched = $matched.Count } $groups = $matched } [ordered]@{ rows = @($result.Rows).Count truncated = $result.Truncated start = ConvertTo-MDOP2IsoDate $range.Start end = ConvertTo-MDOP2IsoDate $range.End content = $content messages = @($groups | ForEach-Object { ConvertTo-MDOP2GroupView -Group $_ }) } } function Invoke-MDOP2PlanJob { param([object]$Payload, [hashtable]$Job) $keys = @($Payload.Keys) Set-MDOP2JobProgress $Job "Locating $($keys.Count) message(s) in the mailboxes" # Always a fresh lookup: a move by anyone gives the message a new Graph ID Update-MDOP2Resolved -Key $keys $destination = Get-MDOP2ActionFolder -Action $Payload.Action $items = [Collections.Generic.List[object]]::new() foreach ($key in $keys) { $g = Get-MDOP2Group -Key $key if (-not $g.InternetMessageId) { $items.Add([pscustomobject]@{ InternetMessageId = $null; Key = $key; Address = $null; UserId = $null; Action = $Payload.Action Destination = $null; GraphMessageId = $null; FolderId = $null; FolderName = $null; Plan = 'Skip'; Reason = 'Message trace has no Message-ID for this message' }) continue } $rows = @($script:Resolved[$key]) # Actions from the Recipients tab of the detail page only touch the chosen mailboxes. Filtered before planning, so a # chosen alias isn't skipped as a duplicate of a primary address that wasn't chosen. if ($Payload.Mailboxes) { $rows = @($rows | Where-Object { $_ -and $_.Address -in $Payload.Mailboxes }) } foreach ($p in New-MDOP2ActionPlan -InternetMessageId $g.InternetMessageId -Resolved $rows -Action $Payload.Action -AllowRestore:([bool]$Payload.Restore)) { $p | Add-Member -NotePropertyName Key -NotePropertyValue $key $items.Add($p) } } # One batched lookup for the folder IDs; it also warms the cache the move itself uses $moves = @($items | Where-Object Plan -EQ 'Move') $map = if ($moves.Count) { Get-MDOP2WellKnownFolderMap -UserId @($moves | ForEach-Object UserId) -Name 'recoverableitemsdeletions', $destination } else { @{} } foreach ($p in $items) { $ri = if ($p.Plan -eq 'Move' -and $map.ContainsKey($p.UserId)) { $map[$p.UserId]['recoverableitemsdeletions'] } else { $null } $p | Add-Member -NotePropertyName InRecoverable -NotePropertyValue ([bool]($ri -and $ri -eq $p.FolderId)) } $id = [guid]::NewGuid().ToString() $script:Plans[$id] = @{ Action = $Payload.Action; Restore = [bool]$Payload.Restore; Items = $items.ToArray(); LiveRun = $false } $s = Get-MDOP2ActionSummary -Plan $items.ToArray() [ordered]@{ planId = $id action = $Payload.Action restore = [bool]$Payload.Restore recoverable = @($items | Where-Object InRecoverable).Count selected = @($Payload.Keys).Count summary = [ordered]@{ messages = $s.Messages; mailboxes = $s.Mailboxes; copies = $s.Copies; skipped = $s.Skipped } items = @($items | ForEach-Object { [ordered]@{ key = $_.Key; mailbox = $_.Address; folder = $_.FolderName; plan = $_.Plan; reason = $_.Reason } }) } } function Invoke-MDOP2ExecuteJob { param([object]$Payload, [hashtable]$Job) $plan = $script:Plans[$Payload.PlanId] if (-not $plan) { throw 'This check has expired. Select the messages and start the action again.' } if ($plan.LiveRun) { throw 'This action already ran. Refresh the status and start a new action if needed.' } if (-not $Payload.DryRun) { $plan.LiveRun = $true } $items = @($plan.Items) $results = [Collections.Generic.List[object]]::new() $i = 0 foreach ($p in $items) { $i++ Set-MDOP2JobProgress $Job "$(if ($Payload.DryRun) { 'Dry run' } else { 'Moving' }) $i of $($items.Count)" # Not Select-Object -First 1: that stops the function before it clears its progress bar $r = @(Invoke-MDOP2MessageAction -Plan @($p) -DryRun $Payload.DryRun)[0] $results.Add([ordered]@{ key = $p.Key; mailbox = $r.Address; fromFolder = $r.FromFolder; result = $r.Result; hold = $r.Hold; error = $r.Error }) } # A move gives the message a new Graph ID, so cached locations are stale after a live run if (-not $Payload.DryRun) { foreach ($k in @($items | ForEach-Object Key | Select-Object -Unique)) { if ($k) { $script:Resolved.Remove($k) } } } [ordered]@{ planId = $Payload.PlanId; action = $plan.Action; dryRun = $Payload.DryRun; results = $results.ToArray() } } function Invoke-MDOP2StatusJob { param([object]$Payload, [hashtable]$Job) $keys = @($Payload.Keys) Set-MDOP2JobProgress $Job "Locating $($keys.Count) message(s) in the mailboxes" Update-MDOP2Resolved -Key $keys $map = Get-MDOP2FolderMapFor -Row @(foreach ($key in $keys) { $script:Resolved[$key] }) $out = foreach ($key in $keys) { $g = Get-MDOP2Group -Key $key if (-not $g.InternetMessageId) { [ordered]@{ key = $key; error = 'Message trace has no Message-ID for this message'; mailboxes = @() } continue } $rows = @($script:Resolved[$key]) # sameAs marks an alias row of a mailbox that another row already stands for, so the UI counts that copy once $sameAs = Get-MDOP2SameMailbox -Row $rows [ordered]@{ key = $key error = $null mailboxes = @(for ($i = 0; $i -lt $rows.Count; $i++) { $r = $rows[$i] [ordered]@{ mailbox = $r.Address; state = $r.State; error = $r.Detail; sameAs = $sameAs[$i]; locations = Get-MDOP2LocationView -Map $map -UserId $r.UserId -Locations $r.Locations } }) } } [ordered]@{ checked = ConvertTo-MDOP2IsoDate ([datetime]::UtcNow); messages = @($out) } } $script:DetailSelect = 'subject,from,sender,replyTo,toRecipients,ccRecipients,receivedDateTime,sentDateTime,internetMessageId,internetMessageHeaders,body,hasAttachments,importance,parentFolderId' $script:InlineImageMaxBytes = 2MB function ConvertTo-MDOP2AddressView { param([AllowNull()][object]$Recipient) if (-not $Recipient -or -not $Recipient.PSObject.Properties['emailAddress'] -or -not $Recipient.emailAddress) { return $null } $e = $Recipient.emailAddress [ordered]@{ name = if ($e.PSObject.Properties['name']) { [string]$e.name } else { '' } address = if ($e.PSObject.Properties['address']) { [string]$e.address } else { '' } } } function Select-MDOP2MessageCopy { <# .SYNOPSIS Picks one mailbox copy of a message to read from: the requested mailbox, else the first mailbox that has it. A cached location is checked again first: a move by ZAP, a user or another admin gives the message a new Graph ID. #> param([string]$Key, [string]$Mailbox) $g = Get-MDOP2Group -Key $Key $pick = { @(foreach ($row in $script:Resolved[$Key]) { if ($row.State -eq 'Found' -and @($row.Locations).Count -and (-not $Mailbox -or $row.Address -eq $Mailbox)) { $row } }) | Select-Object -First 1 } $r = $null if ($script:Resolved.ContainsKey($Key)) { $r = & $pick if ($r) { $now = Find-MDOP2MailboxMessage -UserId $r.UserId -InternetMessageId $g.InternetMessageId $r.State = $now.State $r.Locations = $now.Locations $r.Detail = $now.Error } } if (-not $r -or $r.State -ne 'Found' -or -not @($r.Locations).Count) { Update-MDOP2Resolved -Key $Key $r = & $pick } if (-not $r) { return $null } [pscustomobject]@{ Address = $r.Address UserId = $r.UserId GraphMessageId = $r.Locations[0].GraphMessageId FolderId = $r.Locations[0].FolderId FolderName = $r.Locations[0].FolderName } } function Get-MDOP2AttachmentView { <# .SYNOPSIS Lists attachments with their SHA-256, computed in memory. Only small inline images are returned (as data URIs) so the preview can show embedded CID images; no other attachment content leaves this function. #> param([string]$UserId, [string]$GraphMessageId) $uid = [uri]::EscapeDataString($UserId) $mid = [uri]::EscapeDataString($GraphMessageId) $list = @(Invoke-MDOP2AppGraph -Method GET -Path "/users/$uid/messages/$mid/attachments" -All) $items = [Collections.Generic.List[object]]::new() $inline = [ordered]@{} foreach ($a in $list) { $prop = { param($n) if ($a.PSObject.Properties[$n]) { $a.$n } else { $null } } $type = [string](& $prop '@odata.type') $kind = switch -Wildcard ($type) { '*fileAttachment' { 'File' } '*itemAttachment' { 'Item' } '*referenceAttachment' { 'Reference' } default { 'Other' } } $sha = $null $ctype = [string](& $prop 'contentType') $cid = [string](& $prop 'contentId') $isInline = [bool](& $prop 'isInline') $b64 = & $prop 'contentBytes' if ($kind -eq 'File' -and $b64) { $bytes = [Convert]::FromBase64String([string]$b64) $sha = Get-MDOP2Sha256 -Bytes $bytes # Raster types only (no SVG), the same list the preview accepts for data: images in the body $mime = ($ctype -split ';')[0].Trim().ToLowerInvariant() if ($cid -and $mime -match '^image/(png|jpe?g|gif|webp|bmp)$' -and $bytes.Length -le $script:InlineImageMaxBytes) { $inline[$cid.Trim('<', '>')] = "data:$mime;base64,$b64" } $bytes = $null } $items.Add([ordered]@{ name = [string](& $prop 'name') kind = $kind type = $ctype size = [long](& $prop 'size') isInline = $isInline sha256 = $sha }) } [pscustomobject]@{ Items = $items.ToArray(); Inline = $inline } } function Invoke-MDOP2DetailJob { param([object]$Payload, [hashtable]$Job) $g = Get-MDOP2Group -Key $Payload.Key $view = ConvertTo-MDOP2GroupView -Group $g $out = [ordered]@{ key = $Payload.Key; trace = $view; available = $false; reason = $null; source = $null; message = $null } if (-not $g.InternetMessageId) { $out.reason = 'Message trace has no Message-ID for this message, so it cannot be found in a mailbox.' return $out } Set-MDOP2JobProgress $Job 'Finding a copy in the mailboxes' $copy = Select-MDOP2MessageCopy -Key $Payload.Key -Mailbox $Payload.Mailbox if (-not $copy) { $out.reason = 'No copy of this message is in a mailbox that can be read (for example quarantined, failed or not delivered).' return $out } Set-MDOP2JobProgress $Job "Reading the message from $($copy.Address)" $uid = [uri]::EscapeDataString($copy.UserId) $mid = [uri]::EscapeDataString($copy.GraphMessageId) $m = Invoke-MDOP2AppGraph -Method GET -Path "/users/$uid/messages/$mid`?`$select=$script:DetailSelect" Write-MDOP2Audit -Type View -Data ([ordered]@{ internetMessageId = $g.InternetMessageId; mailbox = $copy.Address; userId = $copy.UserId; folder = $copy.FolderName }) $p = { param($o, $n) if ($o -and $o.PSObject.Properties[$n]) { $o.$n } else { $null } } # Dropping empty values keeps @() empty when Graph leaves the headers out $headers = @(& $p $m 'internetMessageHeaders' | Where-Object { $_ }) $auth = ConvertFrom-MDOP2AuthenticationResult -Value (Get-MDOP2HeaderValue -Headers $headers -Name 'Authentication-Results') $spamValues = Get-MDOP2HeaderValue -Headers $headers -Name 'X-Forefront-Antispam-Report' $spam = ConvertFrom-MDOP2AntispamReport -Value ($spamValues | Select-Object -First 1) $received = ConvertFrom-MDOP2ReceivedHeader -Value (Get-MDOP2HeaderValue -Headers $headers -Name 'Received') $returnPathValues = Get-MDOP2HeaderValue -Headers $headers -Name 'Return-Path' $returnPath = ($returnPathValues | Select-Object -First 1) -as [string] $from = ConvertTo-MDOP2AddressView (& $p $m 'from') $replyTo = @(@(& $p $m 'replyTo') | ForEach-Object { ConvertTo-MDOP2AddressView $_ } | Where-Object { $_ }) $flags = Get-MDOP2SuspicionFlag -FromAddress $(if ($from) { $from.address }) -FromName $(if ($from) { $from.name }) ` -ReturnPath $returnPath -ReplyTo @($replyTo | ForEach-Object { $_.address }) -Authentication $auth Set-MDOP2JobProgress $Job 'Reading attachments' $att = Get-MDOP2AttachmentView -UserId $copy.UserId -GraphMessageId $copy.GraphMessageId $body = & $p $m 'body' $out.available = $true $out.source = [ordered]@{ mailbox = $copy.Address; folder = $copy.FolderName } $out.message = [ordered]@{ subject = [string](& $p $m 'subject') from = $from sender = ConvertTo-MDOP2AddressView (& $p $m 'sender') replyTo = $replyTo returnPath = if ($returnPath) { $returnPath.Trim().Trim('<', '>') } else { '' } to = @(@(& $p $m 'toRecipients') | ForEach-Object { ConvertTo-MDOP2AddressView $_ } | Where-Object { $_ }) cc = @(@(& $p $m 'ccRecipients') | ForEach-Object { ConvertTo-MDOP2AddressView $_ } | Where-Object { $_ }) received = ConvertTo-MDOP2IsoDate (& $p $m 'receivedDateTime') sent = ConvertTo-MDOP2IsoDate (& $p $m 'sentDateTime') importance = [string](& $p $m 'importance') authentication = [ordered]@{ spf = $auth.spf; dkim = $auth.dkim; dmarc = $auth.dmarc; compauth = $auth.compauth; raw = $auth.raw } antispam = [ordered]@{ ip = $spam['CIP']; country = $spam['CTRY']; scl = $spam['SCL']; verdict = $spam['SFV']; category = $spam['CAT'] ptr = $spam['PTR']; helo = $spam['H']; direction = $spam['DIR'] } flags = @($flags | ForEach-Object { [ordered]@{ severity = $_.Severity; text = $_.Text } }) headers = @($headers | ForEach-Object { [ordered]@{ name = [string]$_.name; value = [string]$_.value } }) receivedChain = @($received | ForEach-Object { [ordered]@{ hop = $_.Hop; from = $_.From; by = $_.By; with = $_.With; date = ConvertTo-MDOP2IsoDate $_.Date; delay = $_.DelaySeconds } }) body = [ordered]@{ type = if ($body) { [string](& $p $body 'contentType') } else { 'text' } content = if ($body) { [string](& $p $body 'content') } else { '' } } attachments = $att.Items inline = $att.Inline } $out } function Invoke-MDOP2DownloadJob { param([object]$Payload, [hashtable]$Job) $g = Get-MDOP2Group -Key $Payload.Key if (-not $g.InternetMessageId) { throw 'Message trace has no Message-ID for this message.' } Set-MDOP2JobProgress $Job 'Finding a copy in the mailboxes' $copy = Select-MDOP2MessageCopy -Key $Payload.Key -Mailbox $Payload.Mailbox if (-not $copy) { throw 'No copy of this message is in a mailbox that can be read.' } Set-MDOP2JobProgress $Job "Downloading from $($copy.Address)" $uid = [uri]::EscapeDataString($copy.UserId) $mid = [uri]::EscapeDataString($copy.GraphMessageId) $resp = Invoke-MDOP2AppGraph -Method GET -Path "/users/$uid/messages/$mid/`$value" -Raw $bytes = $resp.RawContentStream.ToArray() $sha = Get-MDOP2Sha256 -Bytes $bytes $name = Get-MDOP2SafeFileName -Subject $g.Subject Write-MDOP2Audit -Type Download -Data ([ordered]@{ internetMessageId = $g.InternetMessageId; mailbox = $copy.Address; userId = $copy.UserId; folder = $copy.FolderName fileName = $name; sizeBytes = $bytes.Length; sha256 = $sha }) # Kept in memory only, on the job, until the browser fetches it once $Job.File = @{ Bytes = $bytes; Name = $name } [ordered]@{ fileName = $name; size = $bytes.Length; sha256 = $sha; mailbox = $copy.Address } } function Invoke-MDOP2TimelineJob { param([object]$Payload, [hashtable]$Job) $g = Get-MDOP2Group -Key $Payload.Key if (-not $g.InternetMessageId) { return [ordered]@{ key = $Payload.Key; events = @() } } $events = @(Get-MDOP2AuditEvent -InternetMessageId $g.InternetMessageId | Sort-Object timestamp -Descending) $p = { param($o, $n) if ($o.PSObject.Properties[$n]) { $o.$n } else { $null } } [ordered]@{ key = $Payload.Key events = @($events | ForEach-Object { [ordered]@{ timestamp = ConvertTo-MDOP2IsoDate $_.timestamp; type = [string]$_.type; admin = & $p $_ 'adminUpn'; mailbox = & $p $_ 'mailbox' action = & $p $_ 'action'; dryRun = & $p $_ 'dryRun'; result = & $p $_ 'result'; fromFolder = & $p $_ 'fromFolder' folder = & $p $_ 'folder'; hold = & $p $_ 'hold'; error = & $p $_ 'error'; fileName = & $p $_ 'fileName' sizeBytes = & $p $_ 'sizeBytes'; sha256 = & $p $_ 'sha256' } }) } } function Invoke-MDOP2SenderHistoryJob { param([object]$Payload, [hashtable]$Job) $g = Get-MDOP2Group -Key $Payload.Key $sender = [string]$g.SenderAddress if (-not $sender -or $sender -notmatch '@') { throw 'This message has no sender address in message trace.' } Set-MDOP2JobProgress $Job "Running message trace for $sender over 90 days" $range = Get-MDOP2DateRange -Range '90d' $r = Search-MDOP2MessageTrace -StartDate $range.Start -EndDate $range.End -Sender $sender -MaxRows 5000 $earlier = @($r.Groups | Where-Object { $_.Key -ne $g.Key -and $_.FirstReceived -lt $g.FirstReceived }) $first = $earlier | Sort-Object FirstReceived | Select-Object -First 1 [ordered]@{ sender = $sender earlier = $earlier.Count firstSeen = if ($first) { ConvertTo-MDOP2IsoDate $first.FirstReceived } else { $null } firstTime = $earlier.Count -eq 0 truncated = $r.Truncated } } function Invoke-MDOP2SimilarJob { <# .SYNOPSIS Finds mail that looks like one message: message trace for the same sender, the same subject and the same sending IP in the days around it, then a comparison of each body with the body of that message. #> param([object]$Payload, [hashtable]$Job) $g = Get-MDOP2Group -Key $Payload.Key $now = [datetime]::UtcNow $centre = ConvertTo-MDOP2UtcTimestamp $g.FirstReceived $oldest = $now.AddDays(-90).AddMinutes(10) $start = $centre.AddDays(-$Payload.Days) if ($start -lt $oldest) { $start = $oldest } $end = $centre.AddDays($Payload.Days) if ($end -gt $now) { $end = $now } if ($end -le $start) { throw 'Message trace only covers the last 90 days.' } # One trace per thing the messages can have in common. A criterion without a usable value is left out. $sender = ([string]$g.SenderAddress).Trim() $subject = ([string]$g.Subject).Trim() $ip = ([string]$g.FromIP).Trim() $searches =[ordered]@{ sender = $null; subject = $null; ip = $null } try { if ($sender -and (ConvertTo-MDOP2AddressFilter -Value $sender).ServerSide) { $searches.sender = @{ Sender = $sender } } } catch { Write-Verbose "No trace on the sender: $($_.Exception.Message)" } if ($subject.Length -ge $script:SimilarSubjectMin -and $subject -notmatch '[\x00-\x1F]') { $searches.subject = @{ Subject = $subject; SubjectFilterType = 'StartsWith' } } if (Test-MDOP2IPAddress $ip) { $searches.ip = @{ FromIP = $ip } } $labels = @{ sender = 'sender'; subject = 'subject'; ip = 'sender IP' } $found = @{} $truncated = $false foreach ($name in @($searches.get_Keys())) { $filter = $searches[$name] if (-not $filter) { continue } Set-MDOP2JobProgress $Job "Running message trace for the same $($labels[$name])" $r = Search-MDOP2MessageTrace -StartDate $start -EndDate $end -MaxRows $script:SimilarTraceRows @filter if ($r.Truncated) { $truncated = $true } foreach ($c in $r.Groups) { if ($c.Key -eq $g.Key -or $found.ContainsKey($c.Key)) { continue } # The reasons come from the message itself, so a subject that only starts the same isn't a match $reasons = Get-MDOP2SimilarReason -Reference $g -Candidate $c if (-not $reasons.Count) { continue } $found[$c.Key] = [pscustomobject]@{ Group = $c Reasons = $reasons Distance = [Math]::Abs(((ConvertTo-MDOP2UtcTimestamp $c.FirstReceived) - $centre).TotalSeconds) Similarity = $null } } } # Most in common first, then closest in time. That order decides what is shown and what is compared. $all = @($found.get_Values() | Sort-Object @{ Expression = { $_.Reasons.Count }; Descending = $true }, @{ Expression = 'Distance' }) $shown = @($all | Select-Object -First $script:SimilarMax) foreach ($s in $shown) { # A message from an earlier search keeps its stored recipients, because actions are planned from the stored message if ($script:Groups.ContainsKey($s.Group.Key)) { $s.Group = $script:Groups[$s.Group.Key] } else { $script:Groups[$s.Group.Key] = $s.Group } } $bodyRead = $false $toCompare = @($shown | Select-Object -First $script:ContentCheckMax) if ($toCompare.Count) { Set-MDOP2JobProgress $Job "Comparing the content of $($toCompare.Count) message(s)" $bodies = Get-MDOP2GroupBody -Group (@($g) + @($toCompare | ForEach-Object { $_.Group })) if ($bodies.ContainsKey($g.Key)) { $reference = Get-MDOP2BodyShingle -Body $bodies[$g.Key] $bodyRead = $reference.Count -gt 0 if ($bodyRead) { foreach ($s in $toCompare) { if ($bodies.ContainsKey($s.Group.Key)) { $s.Similarity = Get-MDOP2ShingleSimilarity -Reference $reference -Other (Get-MDOP2BodyShingle -Body $bodies[$s.Group.Key]) } } } } $bodies = $null } $ordered = @($shown | Sort-Object @{ Expression = { if ($null -eq $_.Similarity) { -1 } else { $_.Similarity } }; Descending = $true }, @{ Expression = { $_.Reasons.Count }; Descending = $true }, @{ Expression = 'Distance' }) [ordered]@{ key = $g.Key days = $Payload.Days start = ConvertTo-MDOP2IsoDate $start end = ConvertTo-MDOP2IsoDate $end criteria = [ordered]@{ sender = if ($searches.sender) { $sender } else { $null } subject = if ($searches.subject) { $subject } else { $null } ip = if ($searches.ip) { $ip } else { $null } } truncated = $truncated total = $all.Count bodyRead = $bodyRead compared = @($shown | Where-Object { $null -ne $_.Similarity }).Count messages = @($ordered | ForEach-Object { $view = ConvertTo-MDOP2GroupView -Group $_.Group $view.reasons = @($_.Reasons) $view.similarity = $_.Similarity $view }) } } function Invoke-MDOP2HistoryJob { param([object]$Payload, [hashtable]$Job) Set-MDOP2JobProgress $Job 'Reading the audit log' $ctx = Get-MDOP2AuditContext $since = if ($Payload.Range -eq 'all') { $null } else { (Get-MDOP2DateRange -Range $Payload.Range).Start } $events = @(Get-MDOP2AuditEvent -Type Action -TenantId $ctx.TenantId) $h = Select-MDOP2ActionHistory -Events $events -Since $since -IncludeDryRun:$Payload.IncludeDryRun ` -SessionId $(if ($Payload.SessionOnly) { $ctx.SessionId }) -Max 5000 # Status checks only accept rows from this list, so the browser can't name arbitrary mailboxes $script:History = @{} $i = 0 $rows = foreach ($r in $h.Rows) { $id = "h$i" $i++ $script:History[$id] = $r [ordered]@{ id = $id; timestamp = ConvertTo-MDOP2IsoDate $r.Timestamp; thisSession = $r.SessionId -eq $ctx.SessionId admin = $r.Admin; internetMessageId = $r.InternetMessageId; mailbox = $r.Mailbox; canCheck = [bool]($r.UserId -and $r.InternetMessageId) action = $r.Action; dryRun = $r.DryRun; fromFolder = $r.FromFolder; result = $r.Result; hold = $r.Hold; error = $r.Error } } [ordered]@{ rows = @($rows); total = $h.Total; truncated = $h.Truncated } } function Invoke-MDOP2HistoryStatusJob { param([object]$Payload, [hashtable]$Job) if (-not $script:History) { throw 'Load the action history first.' } # One lookup per message and mailbox, however many actions ran on it $pairs = [ordered]@{} foreach ($id in $Payload.Ids) { $r = $script:History[$id] if (-not $r) { throw 'The action history changed. Load it again.' } if (-not $r.UserId -or -not $r.InternetMessageId) { continue } $pair = "$($r.InternetMessageId)|$("$($r.Mailbox)".ToLowerInvariant())" if (-not $pairs.Contains($pair)) { $pairs[$pair] = $r } } $names = @($pairs.get_Keys()) $list = @($pairs.get_Values()) Set-MDOP2JobProgress $Job "Locating $($list.Count) message copies" $found = @(if ($list.Count) { Find-MDOP2MessageCopy -Lookup @($list | ForEach-Object { @{ UserId = $_.UserId; InternetMessageId = $_.InternetMessageId } }) }) $rows = @(for ($i = 0; $i -lt $list.Count; $i++) { [pscustomobject]@{ UserId = $list[$i].UserId; State = $found[$i].State } }) $map = Get-MDOP2FolderMapFor -Row $rows $statuses = for ($i = 0; $i -lt $list.Count; $i++) { [ordered]@{ pair = $names[$i] state = $found[$i].State error = $found[$i].Error locations = Get-MDOP2LocationView -Map $map -UserId $list[$i].UserId -Locations $found[$i].Locations } } [ordered]@{ checked = ConvertTo-MDOP2IsoDate ([datetime]::UtcNow); statuses = @($statuses) } } function Invoke-MDOP2MailboxesJob { param([object]$Payload, [hashtable]$Job) Set-MDOP2JobProgress $Job 'Loading mailboxes' $d = Get-MDOP2MailboxDirectory [ordered]@{ truncated = $d.Truncated mailboxes = @($d.Mailboxes | ForEach-Object { [ordered]@{ name = $_.DisplayName; address = $_.Address; upn = $_.Upn; type = $_.Type } }) } } function Invoke-MDOP2Job { <# .SYNOPSIS Runs one queued web job and stores its result or error on the job. #> [CmdletBinding()] param([Parameter(Mandatory)][hashtable]$Job) $Job.State = 'Running' try { $result = switch ($Job.Type) { 'Search' { Invoke-MDOP2SearchJob -Payload $Job.Payload -Job $Job } 'Plan' { Invoke-MDOP2PlanJob -Payload $Job.Payload -Job $Job } 'Execute' { Invoke-MDOP2ExecuteJob -Payload $Job.Payload -Job $Job } 'Status' { Invoke-MDOP2StatusJob -Payload $Job.Payload -Job $Job } 'Detail' { Invoke-MDOP2DetailJob -Payload $Job.Payload -Job $Job } 'Download' { Invoke-MDOP2DownloadJob -Payload $Job.Payload -Job $Job } 'Timeline' { Invoke-MDOP2TimelineJob -Payload $Job.Payload -Job $Job } 'SenderHistory' { Invoke-MDOP2SenderHistoryJob -Payload $Job.Payload -Job $Job } 'Similar' { Invoke-MDOP2SimilarJob -Payload $Job.Payload -Job $Job } 'History' { Invoke-MDOP2HistoryJob -Payload $Job.Payload -Job $Job } 'HistoryStatus' { Invoke-MDOP2HistoryStatusJob -Payload $Job.Payload -Job $Job } 'Mailboxes' { Invoke-MDOP2MailboxesJob -Payload $Job.Payload -Job $Job } default { throw "Unknown job type $($Job.Type)." } } $Job.Result = $result $Job.Progress = 'Done' $Job.State = 'Done' } catch { $at = $_.InvocationInfo $where = if ($at -and $at.ScriptName) { " (at $(Split-Path -Leaf $at.ScriptName):$($at.ScriptLineNumber))" } else { '' } Write-Warning "$($Job.Type) failed: $($_.Exception.Message)$where" $Job.Error = $_.Exception.Message $Job.State = 'Failed' } } Export-ModuleMember -Function Invoke-MDOP2Job |