ExchangeLogs.psm1
|
$script:ModuleRoot = $PSScriptRoot $script:ModuleVersion = (Import-PowerShellDataFile -Path "$($script:ModuleRoot)\ExchangeLogs.psd1").ModuleVersion # Detect whether at some level dotsourcing was enforced $script:doDotSource = Get-PSFConfigValue -FullName ExchangeLogs.Import.DoDotSource -Fallback $false if ($ExchangeLogs_dotsourcemodule) { $script:doDotSource = $true } <# Note on Resolve-Path: All paths are sent through Resolve-Path/Resolve-PSFPath in order to convert them to the correct path separator. This allows ignoring path separators throughout the import sequence, which could otherwise cause trouble depending on OS. Resolve-Path can only be used for paths that already exist, Resolve-PSFPath can accept that the last leaf my not exist. This is important when testing for paths. #> # Detect whether at some level loading individual module files, rather than the compiled module was enforced $importIndividualFiles = Get-PSFConfigValue -FullName ExchangeLogs.Import.IndividualFiles -Fallback $false if ($ExchangeLogs_importIndividualFiles) { $importIndividualFiles = $true } if (Test-Path (Resolve-PSFPath -Path "$($script:ModuleRoot)\..\.git" -SingleItem -NewChild)) { $importIndividualFiles = $true } if ("<was compiled>" -eq '<was not compiled>') { $importIndividualFiles = $true } function Import-ModuleFile { <# .SYNOPSIS Loads files into the module on module import. .DESCRIPTION This helper function is used during module initialization. It should always be dotsourced itself, in order to proper function. This provides a central location to react to files being imported, if later desired .PARAMETER Path The path to the file to load .EXAMPLE PS C:\> . Import-ModuleFile -File $function.FullName Imports the file stored in $function according to import policy #> [CmdletBinding()] Param ( [string] $Path ) $resolvedPath = $ExecutionContext.SessionState.Path.GetResolvedPSPathFromPSPath($Path).ProviderPath if ($doDotSource) { . $resolvedPath } else { $ExecutionContext.InvokeCommand.InvokeScript($false, ([scriptblock]::Create([io.file]::ReadAllText($resolvedPath))), $null, $null) } } #region Load individual files if ($importIndividualFiles) { # Execute Preimport actions foreach ($path in (& "$ModuleRoot\internal\scripts\preimport.ps1")) { . Import-ModuleFile -Path $path } # Import all internal functions foreach ($function in (Get-ChildItem "$ModuleRoot\internal\functions" -Filter "*.ps1" -Recurse -ErrorAction Ignore)) { . Import-ModuleFile -Path $function.FullName } # Import all public functions foreach ($function in (Get-ChildItem "$ModuleRoot\functions" -Filter "*.ps1" -Recurse -ErrorAction Ignore)) { . Import-ModuleFile -Path $function.FullName } # Execute Postimport actions foreach ($path in (& "$ModuleRoot\internal\scripts\postimport.ps1")) { . Import-ModuleFile -Path $path } # End it here, do not load compiled code below return } #endregion Load individual files #region Load compiled code <# This file loads the strings documents from the respective language folders. This allows localizing messages and errors. Load psd1 language files for each language you wish to support. Partial translations are acceptable - when missing a current language message, it will fallback to English or another available language. #> Import-PSFLocalizedString -Path "$($script:ModuleRoot)\en-us\*.psd1" -Module 'ExchangeLogs' -Language 'en-US' function Wait-JobCompleteWithOutput { <# .SYNOPSIS Waits until job(s) are completed and has delivered the output .DESCRIPTION Waits until job(s) are completed and has delivered the output .PARAMETER Job Job(s) to wait for .EXAMPLE PS C:\> Assert-RSJobCompleteWithOutput -Job $jobs Returns true when all jobs are finished #> [CmdletBinding()] param ( $Job ) foreach ($item in $Job) { if($item.State -like "Completed" -and $item.HasMoreData -and -not $item.Output.Count) { Write-PSFMessage -Level Debug -Message "Runspace job '$($item.name)' is in state '$($item.state)' but did not delivered output data. Waiting for output" while (-not $item.Output) { # do nothing, check again. tooks usally arround 50-100ms } } } } function global:Expand-LogRecordPopImap { <# .SYNOPSIS Expand the data from records group into a flat data record .DESCRIPTION Expand the data from records group into a flat data record .PARAMETER InputObject The dataset to expand .PARAMETER SessionIdName The name of the session grouping attribute .PARAMETER ShowProgress If specified, progress information on record processing is showed .PARAMETER IncludeActivityContextData Include full exchange internals in POP3 log expansion process. There is a lot of internal system information in the logfiles, with unneccessary content for statistical output. Suppressing this data is reducing the log footage. .EXAMPLE PS C:\> Expand-LogRecordPop -InputObject $DataSet Expand the data from records group into a flat data record #> [CmdletBinding()] param ( $InputObject, [string] $SessionIdName = "sessionId", [switch] $ShowProgress, [switch] $IncludeActivityContextData ) begin { $Error.Clear() } process { if ($ShowProgress) { $i = 0 if ($InputObject.count -lt 100) { $refreshInterval = 1 } else { $refreshInterval = [math]::Round($InputObject.count / 100) } } foreach ($record in $InputObject) { # assure qualified session in log $_startIndicator = $record.Group[0] | Where-Object command -like "OpenSession" if (-not $_startIndicator) { Write-PSFMessage -Level Warning -Message "Detect fragmented record! Missing previous logfile with partital records. Skip processing $($SessionIdName) '$($record.$SessionIdName)' in $($record.LogFolder)\$($record.LogFileName)" continue } # data text record in array, avoids parsing full data array $commands = $record.Group.command $context = $record.Group.context #| Select-Object -Unique # full text log $logLines = New-Object -TypeName "System.Collections.ArrayList" foreach ($item in $record.Group) { $logline = $item.seqNumber + " " + $item.command if ($item.parameters.Length -gt 0) { $logline = $logline + " | parameters: " + $item.parameters } if ($item.context.Length -gt 0) { if (-not $IncludeActivityContextData) { if ($item.context -like "*;ActivityContextData=*") { $logline = $logline + ([string]::Join(";", ($item.context.Split(";") -notlike "*ActivityContextData=*"))) } else { $logline = $logline + " | context: " + $item.context } } else { $logline = $logline + " | context: " + $item.context } } $null = $logLines.add($logLine) } if ($logLines) { $logText = [string]::Join("`n", ($logLines | ForEach-Object { $_ }) ) } else { $logtext = "" } # LocalEndpoint [string]$localEndpoint = $record.Group[-1].sIP # RemoteEndpoint [string]$remoteEndpoint = $record.Group[-1].cIp # Errors #Log-type - IMAP4 Log $errMsgs = $context | Where-Object { $_ -like 'R="*' -or $_ -like 'ErrMsg=*' } if ($errMsgs) { $hasError = $true $errorMessage = foreach ($errMsg in $errMsgs) { if($errMsg.StartsWith("ErrMsg=")) { $errMsg -replace "ErrMsg=" } elseif($errMsg.StartsWith("R=")) { $_msg = $errMsg.split(";") | Where-Object { $_ -like "R=*" } if($_msg) { (([string]($errMsg.split(";")[0] -replace "R=").Trim('"')) -replace "z NO ") -replace "-ERR " } else { $errMsg } } else { $errMsg } } if ($errorMessage.count -gt 1) { $errorMessage = [string]::Join(" | ", $errorMessage) } } else { $hasError = $false $errorMessage = "" } # IsProxysession $proxycontext = $context -like "*Proxy:*" if ("proxy" -in $commands) { $isProxysession = $true $proxyServer = $record.Group | Where-Object { $_.command -like "proxy" } | Select-Object -ExpandProperty parameters $proxyStatus = "" } elseif ($proxycontext) { $isProxysession = $true $proxycontext = ((($proxycontext -split "Msg=") | Where-Object { $_ -like "*Proxy:*" }) -split '"' | Where-Object { $_ -like "*Proxy:*" }) -replace "Proxy:" $proxyServer = ($proxycontext -split ";")[0] $proxyStatus = ($proxycontext -split ";")[1] } else { $isProxysession = $false $proxyServer = "" $proxyStatus = "" } # Authentication if ("auth" -in $commands -or "user" -in $commands -or "authenticate" -in $commands -or "login" -in $commands) { $authenticationEnabled = $true } else { $authenticationEnabled = $false } if ($authenticationEnabled) { $userName = $record.Group | Where-Object { $_.command -like "auth" -or $_.command -like "user" -or $_.command -like "authenticate" -or $_.command -like "login" } | Select-Object -ExpandProperty User -Unique if ($userName.count -gt 1) { $userName = [string]::Join(", ", $userName) } } else { $userName = "" } if ($authenticationEnabled -and ("auth" -in $commands -or "pass" -in $commands -or "authenticate" -in $commands -or "login" -in $commands)) { $authDetails = ((($record.Group | Where-Object { $_.command -like "auth" -or $_.command -like "pass" -or $_.command -like "authenticate" -or $_.command -like "login"}).context -split ";")[1] -replace 'Msg="').TrimEnd('"') -Split ", " $authHash = @{ "RecipientType" = ($authDetails | Where-Object { $_ -like "RecipientType: *" }) -replace "RecipientType: " "RecipientTypeDetails" = ($authDetails | Where-Object { $_ -like "RecipientTypeDetails: *" }) -replace "RecipientTypeDetails: " "DisplayName" = ($authDetails | Where-Object { $_ -like "Selected Mailbox: Display Name: *" }) -replace "Selected Mailbox: Display Name: " "MailboxGuid" = ($authDetails | Where-Object { $_ -like "Mailbox Guid: *" }) -replace "Mailbox Guid: " "DatabaseGuid" = ($authDetails | Where-Object { $_ -like "Database: *" }) -replace "Database: " "ServerFqdn" = ($authDetails | Where-Object { $_ -like "Location: ServerFqdn: *" }) -replace "Location: ServerFqdn: " "ServerVersion" = ($authDetails | Where-Object { $_ -like "ServerVersion: *" }) -replace "ServerVersion: " "DatabaseName" = ($authDetails | Where-Object { $_ -like "DatabaseName: *" }) -replace "DatabaseName: " "HomePublicFolderDatabaseGuid" = ($authDetails | Where-Object { $_ -like "HomePublicFolderDatabaseGuid: *" }) -replace "HomePublicFolderDatabaseGuid: " } } else { $authDetails = "" $authHash = @{} } # Statistics $stats = (($record.Group | Where-Object { $_.command -like "stat" }).Context -split ";") | Where-Object { $_ -like "Rows=*" -or $_ -like "TotalSize=*" } if ($stats) { $rows = ($stats -like "Rows=*") -replace "Rows=" if ($rows) { $rows = [int]::Parse($rows) } else { [int]$rows = 0 } $totalSize = ($stats -like "TotalSize=*") -replace "TotalSize=" if ($totalSize) { $totalSize = $totalSize = [int]::Parse($totalSize) } else { [int]$totalSize = 0 } } else { $rows = 0 $totalSize = 0 } $budgetinfo = ((([array]$context -like "*Budget=*") | Select-Object -Last 1) -split 'Budget="')[-1] if ($budgetinfo) { $budgetDetails = $budgetinfo.TrimEnd('"') -Split "," $budgetHash = @{ "OwnerSID" = (($budgetDetails | Where-Object { $_ -like "Owner:Sid~*" }) -split "~")[1] "Conn" = [int](($budgetDetails | Where-Object { $_ -like "Conn:*" }) -replace "Conn:") "MaxConn" = ($budgetDetails | Where-Object { $_ -like "MaxConn:*" }) -replace "MaxConn:" "MaxBurst" = ($budgetDetails | Where-Object { $_ -like "MaxBurst:*" }) -replace "MaxBurst:" "Balance" = ($budgetDetails | Where-Object { $_ -like "Balance:*" }) -replace "Balance:" "Cutoff" = ($budgetDetails | Where-Object { $_ -like "Cutoff:*" }) -replace "Cutoff:" "RechargeRate" = ($budgetDetails | Where-Object { $_ -like "RechargeRate:*" }) -replace "RechargeRate:" "Policy" = ($budgetDetails | Where-Object { $_ -like "Policy:*" }) -replace "Policy:" "IsServiceAccount" = [bool]::Parse( (($budgetDetails | Where-Object { $_ -like "IsServiceAccount:*" }) -replace "IsServiceAccount:") ) "LiveTime" = [timespan]::Parse( (($budgetDetails | Where-Object { $_ -like "LiveTime:*" }) -replace "LiveTime:") ) } } else { $budgetDetails = "" $budgetHash = @{} } $stls = ($record.group.command | Where-Object { $_ -like "stls" -or $_ -like "starttls"}).count $auth = ($record.group.command | Where-Object { $_ -like "auth" -or $_ -like "authenticate"}).count $user = ($record.group.command | Where-Object { $_ -like "user" -or $_ -like "login"}).count $pass = ($record.group.command | Where-Object { $_ -like "pass" }).count $stat = ($record.group.command | Where-Object { $_ -like "stat*"}).count $uidl = ($record.group.command | Where-Object { $_ -like "uidl" }).count $list = ($record.group.command | Where-Object { $_ -like "list" }).count $retr = ($record.group.command | Where-Object { $_ -like "retr" }).count $dele = ($record.group.command | Where-Object { $_ -like "dele" }).count # construct output object $outputRecord = [PSCustomObject]@{ "PSTypeName" = "ExchangeLog.$($record.metadataHash['Log-type'].Replace(' ','')).Record" "LogFolder" = $record.LogFolder "LogFileName" = $record.LogFileName $SessionIdName = $record.$SessionIdName "DateStart" = ($record.Group | Sort-Object 'dateTime')[0].'dateTime' -as [datetime] "DateEnd" = ($record.Group | Sort-Object 'dateTime')[-1].'dateTime' -as [datetime] "SequenceCount" = $record.Group.count "LocalIP" = $localEndpoint -replace ":$([string]$localEndpoint.split(":")[-1])", "" "LocalPort" = $localEndpoint.split(":")[-1] "RemoteIP" = $remoteEndpoint -replace ":$([string]$remoteEndpoint.split(":")[-1])", "" "RemotePort" = $remoteEndpoint.split(":")[-1] "AuthenticationEnabled" = $authenticationEnabled "User" = $userName "RecipientType" = (. { if ($authDetails) { $authHash["RecipientType" ] } }) "RecipientTypeDetails" = (. { if ($authDetails) { $authHash["RecipientTypeDetails" ] } }) "DisplayName" = (. { if ($authDetails) { $authHash["DisplayName" ] } }) "MailboxGuid" = (. { if ($authDetails) { $authHash["MailboxGuid" ] } }) "DatabaseGuid" = (. { if ($authDetails) { $authHash["DatabaseGuid" ] } }) "ServerFqdn" = (. { if ($authDetails) { $authHash["ServerFqdn" ] } }) "ServerVersion" = (. { if ($authDetails) { $authHash["ServerVersion" ] } }) "DatabaseName" = (. { if ($authDetails) { $authHash["DatabaseName" ] } }) "HomePublicFolderDatabaseGuid" = (. { if ($authDetails) { $authHash["HomePublicFolderDatabaseGuid"] } }) "OwnerSID" = (. { if ($budgetDetails) { $budgetHash["OwnerSID"] } }) "ConnectionCount" = (. { if ($budgetDetails) { $budgetHash["Conn"] } }) "ConnectionMax" = (. { if ($budgetDetails) { $budgetHash["MaxConn"] } }) "MaxBurst" = (. { if ($budgetDetails) { $budgetHash["MaxBurst"] } }) "Balance" = (. { if ($budgetDetails) { $budgetHash["Balance"] } }) "Cutoff" = (. { if ($budgetDetails) { $budgetHash["Cutoff"] } }) "RechargeRate" = (. { if ($budgetDetails) { $budgetHash["RechargeRate"] } }) "Policy" = (. { if ($budgetDetails) { $budgetHash["Policy"] } }) "IsServiceAccount" = (. { if ($budgetDetails) { $budgetHash["IsServiceAccount"] } }) "LiveTime" = (. { if ($budgetDetails) { $budgetHash["LiveTime"] } }) "TotalObjects" = $rows "TotalSize" = $totalSize "HasError" = $hasError "ErrorMessage" = $errorMessage "IsProxysession" = $isProxysession "ProxyServer" = $proxyServer "ProxyStatus" = $proxyStatus "CmdCountStls" = $stls "CmdCountAuth" = $auth "CmdCountUser" = $user "CmdCountPass" = $pass "CmdCountStat" = $stat "CmdCountUidl" = $uidl "CmdCountList" = $list "CmdCountRetr" = $retr "CmdCountDele" = $dele "LogText" = $logText } # add metadata attributes foreach ($key in $record.metadataHash.Keys) { if ($key -like "Date") { $value = $record.metadataHash[$key] -as [datetime] } else { $value = $record.metadataHash[$key] } $outputRecord | Add-Member -MemberType NoteProperty -Name $key -Value $value -Force } # output data $outputRecord # report in detail if errors occur (for debugging because the processing in operating in runspaces) if ($Error) { Write-Warning "Error detected while processing $($outputRecord.LogFolder)\$($outputRecord.LogFileName) with $($record.$SessionIdName)" $Error.Clear() } # output progress of switch is set (only debugging purpose) if ($ShowProgress) { if (($i % $refreshInterval) -eq 0) { Write-Progress -Activity "Process logfile record " -Status "$($record.LogFileName) - $($SessionIdName): $($record.$SessionIdName) ($($i) / $($InputObject.count))" -PercentComplete ($i / $InputObject.count * 100) } $i = $i + 1 } } } end { } } (Get-Command Expand-LogRecordPopImap).Visibility = "Private" function global:Expand-LogRecordSmtp { <# .SYNOPSIS Expand the data from records group into a flat data record .DESCRIPTION Expand the data from records group into a flat data record .PARAMETER InputObject The dataset to expand .PARAMETER SessionIdName The name of the session grouping attribute .PARAMETER ShowProgress If specified, progress information on record processing is showed .EXAMPLE PS C:\> Expand-LogRecordSmtp -InputObject $DataSet Expand the data from records group into a flat data record #> [CmdletBinding()] param ( $InputObject, [string] $SessionIdName = "session-Id", [switch] $ShowProgress ) begin { $Error.Clear() } process { if ($ShowProgress) { $i = 0 if ($InputObject.count -lt 100) { $refreshInterval = 1 } else { $refreshInterval = [math]::Round($InputObject.count / 100) } } foreach ($record in $InputObject) { # assure qualified session in log $_startIndicator = $record.Group | Where-Object Event -eq "+" $_stopIndicator = $record.Group | Where-Object Event -eq "-" if ((-not $_startIndicator) -or (-not $_stopIndicator)) { Write-PSFMessage -Level Warning -Message "Detect fragmented record! Skip processing $($SessionIdName) '$($record.$SessionIdName)' in $($record.LogFolder)\$($record.LogFileName)" continue } # data text record in array, avoids parsing full data array $groupData = $record.Group.data # full text log $logtext = "" foreach ($item in $record.Group) { $logtext = $logtext + "$(if($logtext){"`n"})" + $item.event + " " + $item.data if ($item.context.Length -gt 0) { $logtext = $logtext + " (context: " + $item.context + ")" } } # ServerName $serverName = $record.Group[0].'connector-id'.split("\")[0] # ServerNameHELO [string]$_serverNameHELO = $groupData | Where-Object { $_ -like "220 * Microsoft*" } | Select-Object -First 1 if ($_serverNameHELO) { [String]$serverNameHELO = $_serverNameHELO.TrimStart("220 ").Split(" ")[0] } else { [String]$serverNameHELO = "" } # ServerOptions $_serverOptions = foreach ($item in $groupData) { if ($item -match "^250\s\s\S+\sHello\s\[\S+]\s(?'ServerOptions'(\S|\s)+)") { $Matches['ServerOptions'] } } if ($_serverOptions) { [string]$serverOptions = [string]::Join(",", $_serverOptions) } else { [string]$serverOptions = "" } # ClientNameHELO $_clientNameHELO = foreach ($item in ($groupData -like "EHLO *" | Select-Object -Unique)) { ([string]$item).trim("EHLO ") } if ($_clientNameHELO) { [string]$clientNameHELO = [string]::Join(",", $_clientNameHELO) } else { [string]$clientNameHELO = "" } # MailFrom [string[]]$_mailFrom = ([array]$groupData -like "MAIL FROM:*") -Split 'MAIL FROM:' | Where-Object { $_ } if ($_mailFrom) { [string[]]$_mailFrom = ($_mailFrom.trim() -Replace '^<', '') | ForEach-Object { $_.split('>')[0] } | Select-Object -Unique | Where-Object { $_ } } if ($_mailFrom) { [string]$mailFrom = [string]::Join(",", $_mailFrom) } else { [string]$mailFrom = "" } # RcptTo [string[]]$_rcptTo = ([array]$groupData -like "RCPT TO:*") -Split 'RCPT TO:' | Where-Object { $_ } if ($_rcptTo) { [string[]]$_rcptTo = ($_rcptTo.trim() -Replace '^<', '') | ForEach-Object { $_.split('>')[0] } | Where-Object { $_ } } if ($_rcptTo) { [string]$rcptTo = [string]::Join(",", $_rcptTo) } else { [string]$rcptTo = "" } # XOOrg [string[]]$_xoorg = foreach ($item in $groupData) { if ($item -match "XOORG=(?'xoorg'\S+)") { $Matches['xoorg'] } } if ($_xoorg) { [string]$xoorg = [string]::Join(",", ($_xoorg.trim() | Select-Object -Unique)) } else { [string]$xoorg = "" } $smtpIdLine = $groupData -match "^250\s2.6.0\s<(?'SmtpId'\S+)" [timespan]$deliveryDuration = [timespan]::new(0) [double]$deliveryBandwidth = 0 [string]$remoteServerHostName = "" [string]$internalId = "" [int]$mailSize = 0 [String]$SmtpId = "" if ($smtpIdLine) { [string[]]$_smtpIdRecords = foreach ($line in $smtpIdLine) { $line.trim("250 2.6.0 <").split(">")[0] } if ($_smtpIdRecords) { $SmtpId = [string]::Join(",", $_smtpIdRecords.trim() ) } else { [string]$smtpId = "" } [string[]]$_remoteServerHostName = foreach ($item in $_smtpIdRecords) { $item.split("@")[1] } if ($_remoteServerHostName) { [string]$remoteServerHostName = [string]::Join(",", ($_remoteServerHostName | Select-Object -Unique) ) } else { [string]$remoteServerHostName = "" } [string[]]$_internalId = $smtpIdLine | ForEach-Object { ($_ -split "InternalId=")[1].split(",")[0] } if ($_internalId) { [string]$internalId = [string]::Join(",", $_internalId.trim() ) } else { [string]$internalId = "" } if ($smtpIdLine -like "*bytes in*") { [int[]]$_mailSize = $smtpIdLine | ForEach-Object { ($_ -split " bytes in ")[0].split(" ")[-1] } if ($_mailSize) { $mailSize = ($_mailSize | Measure-Object -Sum).Sum } else { [int]$mailSize = 0 } ForEach ($item in $smtpIdLine) { $deliveryDuration = $deliveryDuration + [timespan]::FromSeconds( [System.Convert]::ToDouble( (($item -split " bytes in ")[1].split(", ")[0]) , [cultureinfo]::GetCultureInfo('en-us') )) } ForEach ($item in $smtpIdLine) { $deliveryBandwidth = $deliveryBandwidth + [double]::Parse( (($item.TrimEnd(" KB/sec Queued mail for delivery") -split ", ")[-1]) ) $deliveryBandwidth = [math]::Round( ($deliveryBandwidth / $smtpIdLine.count), 3 ) } } } if ($record.Group | Where-Object data -like "Tarpit*") { $tarpitDetect = $true } else { $tarpitDetect = $false } $tarpitDuration = [timespan]::new(0) $tarpitMessage = "" if ($tarpitDetect) { $tarpitDuration = [timespan]::FromSeconds( (($record.Group | where-Object data -like "Tarpit*").data.replace("Tarpit for '", "") | ForEach-Object { $_.split("'")[0] -as [timespan] } | Measure-Object Seconds -Sum).Sum ) $tarpitMessage = (($record.Group | Where-Object data -like "Tarpit*").data -split "(due\sto\s')")[-1].trim("'") } [string]$connectorID = $record.Group[0].'connector-id' if ($connectorID) { if ($connectorID -match "\\") { $connectorName = $connectorID.split("\")[1] } else { $connectorName = $connectorID } } else { $connectorName = "" } if ($connectorID) { $connectorNameWithoutServerName = $connectorName.replace($serverName, "").trim() } else { $connectorNameWithoutServerName = "" } [string]$localEndpoint = $record.Group[-1].'local-endpoint' [string]$remoteEndpoint = $record.Group[-1].'remote-endpoint' if ($groupData -clike "AUTH *") { $authenticationEnabled = $true } else { $authenticationEnabled = $false } $authenticationType = "" $authenticationUser = "" $authenticationMessage = "" if ($authenticationEnabled) { $null = $record.Group | Where-Object data -Match "^AUTH\s(?'Method'\S+)" $authenticationType = $Matches['Method'] $authenticationUser = ($record.Group | Where-Object context -like "authenticated").data $text = @( "235 2.7.0 Authentication successful", "504 5.7.4 Unrecognized authentication type", "535 5.7.3 Authentication unsuccessful" ) [string]$authenticationMessage = ($record.Group | Where-Object data -in $text)[-1].data } # TLS records if ($groupData -clike " CN=*") { $tlsEnabled = $true } else { $tlsEnabled = $false } $tlsAlgorithmEncryption = "" $tlsAlgorithmKeyExchange = "" $tlsAlgorithmMacHash = "" $tlsCertificateRemote = "" $tlsCertificateRemoteIssuer = "" $tlsCertificateRemoteNotAfter = "" $tlsCertificateRemoteNotBefore = "" $tlsCertificateRemoteSAN = "" $tlsCertificateRemoteSerial = "" $tlsCertificateRemoteThumbprint = "" $TlsCertificateServer = "" $tlsCertificateServerIssuer = "" $tlsCertificateServerNotAfter = "" $tlsCertificateServerNotBefore = "" $tlsCertificateServerSAN = "" $tlsCertificateServerSerial = "" $tlsCertificateServerThumbprint = "" $tlsCrypto = "" $tlsDomain = "" $tlsDomainCapabilities = "" $tlsProtocol = "" $tlsStatus = "" $tlsStatusRecord = "" if ($tlsEnabled) { # gather TLS related records $tlsRecords = $record.Group | Where-Object { ($_.event -eq '*') -and ($_.context -like "Sending certificate*" -or $_.context -like "Remote certificate*" -or $_.context -like "TLS protocol*" -or $_.context -like "*TlsDomainCapabilities=*") } | Select-Object 'sequence-number', context, data if ($tlsRecords) { # TLS Crypto string $_tlsCrypto = $TlsRecords | Where-Object context -like "TLS *" | Select-Object -ExpandProperty context -Unique if ($_tlsCrypto) { $tlsCrypto = [string]::Join(",", $_tlsCrypto) } else { $tlsCrypto = "" } if ($tlsCrypto) { # TLS protocol $_tlsProtocol = foreach ($item in $tlsCrypto) { ([string]$item).Replace('TLS protocol ', '').Split(" ")[0] } if ($_tlsProtocol) { $tlsProtocol = [string]::Join(",", $_tlsProtocol) } else { $tlsProtocol = "" } # TLS Algorithm Encryption $_tlsAlgorithmEncryption = foreach ($item in $tlsCrypto) { ([string](([string]$item) -Split "encryption algorithm ")[1]).Split(" ")[0] } if ($_tlsAlgorithmEncryption) { $tlsAlgorithmEncryption = [string]::Join(",", $_tlsAlgorithmEncryption) } else { $tlsAlgorithmEncryption = "" } # TLS Algorithm MacHash $_tlsAlgorithmMacHash = foreach ($item in $tlsCrypto) { ([string](([string]$item) -Split "hash algorithm ")[1]).Split(" ")[0] } if ($_tlsAlgorithmMacHash) { $tlsAlgorithmMacHash = [string]::Join(",", $_tlsAlgorithmMacHash) } else { $tlsAlgorithmMacHash = "" } # TLS Algorithm KeyExchange $_tlsAlgorithmKeyExchange = foreach ($item in $tlsCrypto) { ([string](([string]$item) -Split "exchange algorithm ")[1]).Split(" ")[0] } if ($_tlsAlgorithmKeyExchange) { $tlsAlgorithmKeyExchange = [string]::Join(",", $_tlsAlgorithmKeyExchange) } else { $tlsAlgorithmKeyExchange = "" } } # TLS Server certificate $_tlsCertificateServerRecord = $tlsRecords | where-Object context -Like "Sending certificat*" | Select-Object -First 1 -ExpandProperty data if ($_tlsCertificateServerRecord) { $_tlsCertificateServerRecord = $_tlsCertificateServerRecord.trim() [string]$certText = $_tlsCertificateServerRecord # TLS Server certificate - Subject alternate names [String]$tlsCertificateServerSAN = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateServerSAN).TrimEnd() # TLS Server certificate - Not after [String]$_tlsCertificateServerNotAfter = $certText.split(" ")[-1] $tlsCertificateServerNotAfter = [datetime]::Parse($_tlsCertificateServerNotAfter) $certText = $certText.TrimEnd($_tlsCertificateServerNotAfter).TrimEnd() # TLS Server certificate - Not before [String]$_tlsCertificateServerNotBefore = $certText.split(" ")[-1] $tlsCertificateServerNotBefore = [datetime]::Parse($_tlsCertificateServerNotBefore) $certText = $certText.TrimEnd($_tlsCertificateServerNotBefore).TrimEnd() # TLS Server certificate - Thumbprint [String]$tlsCertificateServerThumbprint = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateServerThumbprint).TrimEnd() # TLS Server certificate - Serial number [String]$tlsCertificateServerSerial = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateServerSerial).TrimEnd() # TLS Server certificate - Issuer name [String]$tlsCertificateServerIssuer = "CN=" + ($certText -split (" CN="))[1] $certText = $certText.TrimEnd($tlsCertificateServerIssuer).TrimEnd() # TLS Server certificate - Subject [String]$tlsCertificateServerIssuer = $certText } [String]$tlsCertificateServer = $_tlsCertificateServerRecord # TLS Remote certificate $_tlsCertificateRemoteRecord = $tlsRecords | where-Object context -Like "Remote certificat*" | Select-Object -First 1 -ExpandProperty data if ($_tlsCertificateRemoteRecord) { $_tlsCertificateRemoteRecord = $_tlsCertificateRemoteRecord.trim() [string]$certText = $_tlsCertificateRemoteRecord # TLS Remote certificate - Subject alternate names [String]$tlsCertificateRemoteSAN = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateRemoteSAN).TrimEnd() # TLS Remote certificate - Not after [String]$_tlsCertificateRemoteNotAfter = $certText.split(" ")[-1] $tlsCertificateRemoteNotAfter = [datetime]::Parse($_tlsCertificateRemoteNotAfter) $certText = $certText.TrimEnd($_tlsCertificateRemoteNotAfter).TrimEnd() # TLS Remote certificate - Not before [String]$_tlsCertificateRemoteNotBefore = $certText.split(" ")[-1] $tlsCertificateRemoteNotBefore = [datetime]::Parse($_tlsCertificateRemoteNotBefore) $certText = $certText.TrimEnd($_tlsCertificateRemoteNotBefore).TrimEnd() # TLS Remote certificate - Thumbprint [String]$tlsCertificateRemoteThumbprint = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateRemoteThumbprint).TrimEnd() # TLS Remote certificate - Serial number [String]$tlsCertificateRemoteSerial = $certText.split(" ")[-1] $certText = $certText.TrimEnd($tlsCertificateRemoteSerial).TrimEnd() # TLS Remote certificate - Issuer name [String]$tlsCertificateRemoteIssuer = "CN=" + ($certText -split (" CN="))[1] $certText = $certText.TrimEnd($tlsCertificateRemoteIssuer).TrimEnd() # TLS Remote certificate - Subject [String]$tlsCertificateRemoteIssuer = $certText } [String]$tlsCertificateRemote = $_tlsCertificateRemoteRecord # TLS Status Record $_tlsStatusRecord = ($tlsRecords | Where-Object context -Like "*; Status='*").context -Split "; " if ($_tlsStatusRecord) { # TLS Status Record [String]$tlsStatusRecord = [string]::Join("; ", $_tlsStatusRecord) # TLS Domain Capabilities $_tlsDomainCapabilities = ("" + (($_tlsStatusRecord | Where-Object { $_ -like "TlsDomainCapabilities=*" }) -split "=")[1] ).trim("'") if ($_tlsDomainCapabilities) { $tlsDomainCapabilities = [string]::Join(",", $_tlsDomainCapabilities) } else { $tlsDomainCapabilities = "" } # TLS Status $_tlsStatus = ("" + (($_tlsStatusRecord | Where-Object { $_ -like "Status=*" }) -split "=")[1] ).trim("'") if ($_tlsStatus) { $tlsStatus = [string]::Join(",", $_tlsStatus) } else { $tlsStatus = "" } # TLS Domain $_tlsDomain = ("" + (($_tlsStatusRecord | Where-Object { $_ -like "Domain=*" }) -split "=")[1] ).trim("'") if ($_tlsDomain) { $tlsDomain = [string]::Join(",", $_tlsDomain) } else { $tlsDomain = "" } } else { [String]$_tlsStatusRecord = "" } } } # final status $_finalStatus = $record.Group[-1].context if ($_finalStatus -like "Local") { $finalStatus = "OK" } elseif ($_finalStatus) { $finalStatus = $_finalStatus } else { $finalStatus = "UNKNOWN" } # construct output object $outputRecord = [PSCustomObject]@{ "PSTypeName" = "ExchangeLog.$($record.metadataHash['Log-type'].Replace(' ','')).Record" "LogFolder" = $record.LogFolder "LogFileName" = $record.LogFileName $SessionIdName = $record.$SessionIdName "DateStart" = ($record.Group | Sort-Object 'date-time')[0].'date-time' -as [datetime] "DateEnd" = ($record.Group | Sort-Object 'date-time')[-1].'date-time' -as [datetime] "SequenceCount" = $record.Group.count "ConnectorID" = $ConnectorID "ServerName" = $ServerName "ConnectorName" = $ConnectorName "ConnectorNameWithoutServerName" = $ConnectorNameWithoutServerName "LocalIP" = $localEndpoint -replace ":$([string]$localEndpoint.split(":")[-1])", "" "LocalPort" = $localEndpoint.split(":")[-1] "RemoteIP" = $remoteEndpoint -replace ":$([string]$remoteEndpoint.split(":")[-1])", "" "RemotePort" = $remoteEndpoint.split(":")[-1] "ServerNameHELO" = $ServerNameHELO "ServerOptions" = $ServerOptions "ClientNameHELO" = $clientNameHELO "TlsEnabled" = $TlsEnabled "AuthenticationEnabled" = $AuthenticationEnabled "AuthenticationType" = $AuthenticationType "AuthenticationUser" = $AuthenticationUser "AuthenticationMessage" = $AuthenticationMessage "TarpitDetect" = $TarpitDetect "TarpitDuration" = $TarpitDuration "TarpitMessage" = $TarpitMessage "MailFrom" = $MailFrom "RcptTo" = $rcptTo "XOOrg" = $XOOrg "SmtpId" = $SmtpId "RemoteServerHostName" = $RemoteServerHostName "InternalId" = $InternalId "MailSize" = $MailSize "DeliveryDuration" = $DeliveryDuration "DeliveryBandwidth" = $deliveryBandwidth "FinalSessionStatus" = $finalStatus "FinalizeMessage" = $groupData[-2] "TlsProtocol" = $tlsProtocol "TlsAlgorithmEncryption" = $tlsAlgorithmEncryption "TlsAlgorithmMacHash" = $tlsAlgorithmMacHash "TlsAlgorithmKeyExchange" = $tlsAlgorithmKeyExchange "TlsCertificateServer" = $tlsCertificateServer "TlsCertificateServerIssuer" = $tlsCertificateServerIssuer "TlsCertificateServerNotAfter" = $tlsCertificateServerNotAfter "TlsCertificateServerNotBefore" = $tlsCertificateServerNotBefore "TlsCertificateServerSAN" = $tlsCertificateServerSAN "TlsCertificateServerSerial" = $tlsCertificateServerSerial "TlsCertificateServerThumbprint" = $tlsCertificateServerThumbprint "TlsCertificateRemote" = $tlsCertificateRemote "TlsCertificateRemoteIssuer" = $tlsCertificateRemoteIssuer "TlsCertificateRemoteNotAfter" = $tlsCertificateRemoteNotAfter "TlsCertificateRemoteNotBefore" = $tlsCertificateRemoteNotBefore "TlsCertificateRemoteSAN" = $tlsCertificateRemoteSAN "TlsCertificateRemoteSerial" = $tlsCertificateRemoteSerial "TlsCertificateRemoteThumbprint" = $tlsCertificateRemoteThumbprint "TlsStatus" = $tlsStatus "TlsDomain" = $tlsDomain "TlsDomainCapabilities" = $tlsDomainCapabilities "TlsCrypto" = $tlsCrypto "TlsStatusRecord" = $tlsStatusRecord "LogText" = $logText } # add metadata attributes foreach ($key in $record.metadataHash.Keys) { if ($key -like "Date") { $value = $record.metadataHash[$key] -as [datetime] } else { $value = $record.metadataHash[$key] } $outputRecord | Add-Member -MemberType NoteProperty -Name $key -Value $value -Force } # output data $outputRecord # report in detail if errors occur (for debugging because the processing in operating in runspaces) if ($Error) { Write-Warning "Error detected while processing $($outputRecord.LogFolder)\$($outputRecord.LogFileName) with $($record.$SessionIdName)" $Error.Clear() } # output progress of switch is set (only debugging purpose) if ($ShowProgress) { if (($i % $refreshInterval) -eq 0) { Write-Progress -Activity "Process logfile record " -Status "$($record.LogFileName) - $($SessionIdName): $($record.$SessionIdName) ($($i) / $($InputObject.count))" -PercentComplete ($i / $InputObject.count * 100) } $i = $i + 1 } } } end { } } (Get-Command Expand-LogRecordSmtp).Visibility = "Private" function Import-LogData { <# .SYNOPSIS Get csv content from logfile output grouped records .DESCRIPTION Get csv content from logfile output grouped records .PARAMETER File The file to gather data .EXAMPLE PS C:\> Import-LogData -File RECV2020061300-1.LOG Return the csv records as grouped records by SessionID #> [CmdletBinding()] [OutputType([System.Collections.ArrayList])] param ( $File ) # Get content from logfile $File = Get-Item -Path $File -ErrorAction Stop Write-PSFMessage -Level Verbose -Message "Get content from logfile: $($File.Fullname)" $content = $File | Get-Content # split metadata and logcontent (first lines fo logfile) $metadata = $content -match '^#.*' $header = $metadata[-1].split(": ")[-1] $logcontent = $content -notmatch $header # query meta data informations into hashtable $metadataHash = [ordered]@{} foreach ($metadatarecord in ($metadata[0 .. ($metadata.Count - 2)])) { $_data = $metadatarecord.TrimStart("#") -Split ': ' $metadataHash.Add($_data[0], $_data[1]) } Write-PSFMessage -Level VeryVerbose -Message "Detect $($metadataHash['Log-type'])" # convert filecontent to csv data and group records if know/supportet logfile type $records = $logcontent | ConvertFrom-Csv -Delimiter "," -Header $header.Split(",") $sessionIdName = Resolve-SessionIdName -LogType $metadataHash['Log-type'] $output = New-Object -TypeName "System.Collections.ArrayList" if ($sessionIdName) { Write-PSFMessage -Level VeryVerbose -Message "Going to group records by $($sessionIdName) from file '$($File.Fullname)'" foreach ($group in ($records | Group-Object $sessionIdName)) { $null = $output.Add( [PSCustomObject]@{ "PSTypeName" = "ExchangeLog.$($metadataHash['Log-type'].Replace(' ','')).Group" $sessionIdName = $group.Name "Group" = $group.Group "Log-type" = $metadataHash["Log-type"] "metadataHash" = $metadataHash "LogFileName" = $File.Name "LogFolder" = $File.Directory } ) } } else { foreach ($record in $records) { $record.PSOBject.TypeNames.Insert(0, "ExchangeLog.$($metadataHash['Log-type'].Replace(' ','')).Record" ) $null = $output.Add( $record ) } # add metadata info to record groups foreach ($key in $metadataHash.Keys) { $output | Add-Member -MemberType NoteProperty -Name $key -Value $metadataHash[$key] -Force } $output | Add-Member -MemberType NoteProperty -Name "LogFileName" -Value $File.Name -Force $output | Add-Member -MemberType NoteProperty -Name "LogFolder" -Value $File.Directory -Force } Write-PSFMessage -Level VeryVerbose -Message "Finished processing $($output.count) records from file '$($File.Name)'" # output data to the pipeline $output } function Import-LogData_typed { <# .SYNOPSIS Get csv content from logfile output grouped records .DESCRIPTION Get csv content from logfile output grouped records .PARAMETER File The file to gather data .EXAMPLE PS C:\> Import-LogData -File RECV2020061300-1.LOG Return the csv records as grouped records by SessionID #> [CmdletBinding()] [OutputType([System.Collections.ArrayList])] param ( $File ) # Get content from logfile $File = Get-ChildItem -Path $File -File -ErrorAction Stop Write-PSFMessage -Level Verbose -Message "Get content from logfile: $($File.Fullname)" $content = $File | Get-Content # split metadata and logcontent (first lines fo logfile) $metadata = $content -match '^#.*' $header = $metadata[-1].split(": ")[-1] $logcontent = $content -notmatch $header # query meta data informations into hashtable $metadataHash = [ordered]@{} foreach ($metadatarecord in ($metadata[0 .. ($metadata.Count - 2)])) { $_data = $metadatarecord.TrimStart("#") -Split ': ' $metadataHash.Add($_data[0], $_data[1]) } # convert filecontent to csv data and group records if know/supportet logfile type $records = $logcontent | ConvertFrom-Csv -Delimiter "," -Header $header.Split(",") $sessionIdName = Resolve-SessionIdName -LogType $metadataHash['Log-type'] $output = New-Object -TypeName "System.Collections.ArrayList" if($sessionIdName) { foreach ($group in ($records | Group-Object $sessionIdName)) { $null = $output.Add( [PSCustomObject]@{ "PSTypeName" = "ExchangeLog.$($metadataHash['Log-type'].Replace(' ','')).TypedRecord" $sessionIdName = $group.Name "Group" = $group.Group } ) } } else { foreach ($record in $records) { $record.PSOBject.TypeNames.Insert(0, "ExchangeLog.$($metadataHash['Log-type'].Replace(' ','')).Record" ) $null = $output.Add( $record ) } } # add metadata info to record groups foreach ($key in $metadataHash.Keys) { $output | Add-Member -MemberType NoteProperty -Name $key -Value $metadataHash[$key] -Force } $output | Add-Member -MemberType NoteProperty -Name "LogFileName" -Value $File.Name -Force $output | Add-Member -MemberType NoteProperty -Name "LogFolder" -Value $File.Directory -Force Write-PSFMessage -Level VeryVerbose -Message "Finished logfile $($File.Name). Found $($output.count) recors" # output data to the pipeline $output } function Resolve-SessionIdName { <# .SYNOPSIS Converts access tokens to readable objects .DESCRIPTION Converts access tokens to readable objects .PARAMETER LogType Name of the LogType from the meta data line in a logfile .EXAMPLE PS C:\> Resolve-SessionIdName -LogType "SMTP Receive Protocol Log" Returns the name of the grouping field for building record groups #> [CmdletBinding()] [OutputType([System.String])] param ( $LogType ) switch ($LogType) { {$_ -in "SMTP Receive Protocol Log", "SMTP Send Protocol Log"} { "session-Id" } {$_ -in "IMAP4 Log", "POP3 Log"} { "sessionId" } {$_ -in "Message Tracking Log"} { "message-id" } Default { Write-Warning "Unknown LogType"} } } function Get-ELExchangeLog { <# .SYNOPSIS Get records from exchange logfiles .DESCRIPTION Get records from exchange logfiles like SMTP Receive / SMTP Send / IMAP / POP /... Files are parsed and grouped by sessions as possible. .PARAMETER Path The folder to gather logfiles .PARAMETER Recurse If specified, the path will be gathered recursive .PARAMETER Filter Filter to be applied for files to parse .PARAMETER LogType Specifies the type of logfile to work through. There are multiple types supported and usually the command does an autodectect. Use tab completion on the parameter to see the supported logfile types. Use this parameter of your workload expects an explicit type of log (for example "SMTPReceiveProtocolLog") and you want to ensure, that no other logfiles are processed. .EXAMPLE PS C:\> Get-ELExchangeLog -Path "C:\Logs\SMTPReceive" Return records from all files in the folder .EXAMPLE PS C:\> Get-ELExchangeLog -Path "C:\Logs\SMTPReceive" -Recursive Return records from all files in the current and all subfolders. #> [CmdletBinding()] [Alias('gel')] param ( [Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true, ValueFromPipelineByPropertyName = $true)] [Alias('FullName')] [String[]] $Path, [switch] $Recurse, [String] $Filter = "*.log", [ValidateSet("AutoDetect", "SMTPReceiveProtocolLog", "SMTPSendProtocolLog", "IMAP4Log", "POP3Log","MessageTrackingLog")] [string] $LogType = "AutoDetect" ) begin { $files = New-Object -TypeName "System.Collections.ArrayList" $batchJobId = $([guid]::NewGuid().ToString()) Write-PSFMessage -Level VeryVerbose -Message "Starting BatchId '$($batchJobId)' with LogType '$LogType'" } process { # get files from folder Write-PSFMessage -Level Verbose -Message "Gettings files$( if($Filter){" by filter '$($Filter)'"} ) in path '$path'" foreach ($pathItem in $Path) { $options = @{ "Path" = $pathItem "File" = $true "ErrorAction" = "Stop" } if ($Recurse) { $options.Add("Recurse", $true) } try { $ChildItemList = Get-ChildItem @options if ($Filter) { $ChildItemList = $ChildItemList | Where-Object Name -like $Filter } ($ChildItemList).FullName | Sort-Object | ForEach-Object { [void]$files.Add( $_ ) } } catch { Stop-PSFFunction -Message "Error, path '$($pathItem)' not found" } } } end { if (-not $files) { Stop-PSFFunction -Message "No file found to parse! ($([string]::Join(", ", $Path)))" break } $recordCount = 0 $files = $files | Sort-Object if ($files.count -lt 100) { $refreshInterval = 1 } else { $refreshInterval = [math]::Round($files.count / 100) } Write-PSFMessage -Level Verbose -Message "$($files.count) file$(if($files.count -gt 1){"s"}) to process." $traceTimer = New-Object System.Diagnostics.Stopwatch $traceTimer.Start() # Import first file if ($files.Count -gt 1) { $filePrevious = $files[0] } else { $filePrevious = $files } $resultPreviousFile = New-Object -TypeName "System.Collections.ArrayList" foreach ($record in (Import-LogData -File $filePrevious)) { [void]$resultPreviousFile.Add($record) } $sessionIdName = Resolve-SessionIdName -LogType $resultPreviousFile[0].'Log-type' # file validity check if($LogType -ne "AutoDetect") { if($LogType -notlike $resultPreviousFile[0].'Log-type'.Replace(" ", "")) { Stop-PSFFunction -Message "Invalid LogType detected/specified. Expect '$($LogType)', but found '$( $resultPreviousFile[0].'Log-type'.Replace(' ', '') )' in file '$($filePrevious)'." break } } else { $LogType = $resultPreviousFile[0].'Log-type'.Replace(" ", "") } if($LogType -notin (Get-PSFConfigValue -FullName 'ExchangeLogs.SupportedLogTypes')) { Stop-PSFFunction -Message "Invalid LogType detected. '$( $resultPreviousFile[0].'Log-type'.Replace(' ', '') )' from file '$($filePrevious)' is not a supported log type to process with this command." break } # Import remaining files Write-PSFMessage -Level Verbose -Message "Starting import on $($files.Count) remaining file(s)." for ($filecounter = 1; $filecounter -lt $files.Count; $filecounter++) { #region process # import next file $fileCurrent = $files[$filecounter] $resultCurrentFile = New-Object -TypeName "System.Collections.ArrayList" foreach ($record in (Import-LogData -File $fileCurrent)) { [void]$resultCurrentFile.Add($record) } if ($resultCurrentFile[0].'Log-type' -ne $resultPreviousFile[0].'Log-type') { Stop-PSFFunction -Message "Incompatible logfile types ($($resultCurrentFile[0].'Log-type'), $($resultPreviousFile[0].'Log-type')) found! More then one type of logfile in folder '$($pathItem)'." break } if ($sessionIdName) { # loop through previous and current file to check on fragmented session records in both files (sessions over midnight) Write-PSFMessage -Level VeryVerbose -Message "Checking for overlapping log records on identifier '$($sessionIdName)' in file '$($filePrevious)' and '$($fileCurrent)'" $overlapSessionIDs = Compare-Object -ReferenceObject $resultPreviousFile[-1..-20] -DifferenceObject $resultCurrentFile[0..20] -Property $sessionIdName -ExcludeDifferent -IncludeEqual if ($overlapSessionIDs) { foreach ($overlapSessionId in $overlapSessionIDs.$sessionIdName) { Write-PSFMessage -Level VeryVerbose -Message "Found overlapping log record '$($overlapSessionId)'" # get the record fragments from both logfiles $overlapRecordCurrentFile = $resultCurrentFile | Where-Object $sessionIdName -like $overlapSessionId $overlapRecordPreviousFile = $resultPreviousFile | Where-Object $sessionIdName -like $overlapSessionId # merge records $overlapRecordPreviousFileMerged = $overlapRecordPreviousFile $overlapRecordPreviousFileMerged.Group = $overlapRecordPreviousFile.Group + $overlapRecordCurrentFile.Group # remove overlapping records from current logfile $resultCurrentFile.RemoveAt( $resultCurrentFile.IndexOf($overlapRecordCurrentFile) ) # remove overlapping records from previous logfile $resultPreviousFile.RemoveAt( $resultPreviousFile.IndexOf($overlapRecordPreviousFile) ) # add merged record into previous logfile [void]$resultPreviousFile.Add($overlapRecordPreviousFileMerged) } } } # invoke data transform processing in a runspace to parallize processing and continue to work through files $recordCount = $recordCount + $resultPreviousFile.count switch ($LogType) { {$_ -in @("SMTPReceiveProtocolLog", "SMTPSendProtocolLog")} { $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordSmtp -Verbose:$false -ScriptBlock { Expand-LogRecordSmtp -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "IMAP4Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "POP3Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "MessageTrackingLog" { Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." } Default { Write-PSFMessage -Level Warning -Message "Unknown LogType: $($LogType) | Probably developers mistake." } } Write-PSFMessage -Level Verbose -Message "Start runspace job '$($jobObject.Name)' (ID:$($jobObject.ID)) for processing $($resultPreviousFile.count) record(s)" # put current file records in variable for previous file to check for further record fragments $filePrevious = $fileCurrent $resultPreviousFile = $resultCurrentFile # progress status info & receive completed runspaces if (($filecounter % $refreshInterval) -eq 0 -or $filecounter -eq $files.count) { Write-PSFMessage -Level System -Message "Procesed $refreshInterval files... Going to update progress status" $jobs = Get-RSJob -Batch $batchJobId -ErrorAction SilentlyContinue -Verbose:$false | Sort-Object ID $jobsCompleted = $jobs | Where-Object State -Like "Completed" # output remaining data in completed runspace if ($jobsCompleted) { Wait-JobCompleteWithOutput -Job $jobsCompleted $recordsProcessed = Receive-RSJob -Job $jobsCompleted -Verbose:$false foreach ($recordProcessed in $recordsProcessed) { [void]$recordProcessed.psobject.TypeNames.Remove('Selected.RSJob') $recordProcessed } Write-PSFMessage -Level VeryVerbose -Message "Receiving $($jobsCompleted.Count) completed runspace job(s) with $($recordsProcessed.count) processed records" Remove-RSJob -Job $jobsCompleted -Verbose:$false } # status update Write-Progress -Activity "Import logfiles in $($resultPreviousFile[0].LogFolder) | Currently working runspaces: $($jobs.count - $jobsCompleted.count) | Records in processing: $($recordCount) | Time elapsed: $($traceTimer.Elapsed)" -Status "$($resultPreviousFile[0].LogFileName) ($($filecounter) / $($files.count))" -PercentComplete ($filecounter / $files.count * 100) } #endregion process } # processing last remaining file switch ($LogType) { {$_ -in @("SMTPReceiveProtocolLog", "SMTPSendProtocolLog")} { $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordSmtp -Verbose:$false -ScriptBlock { Expand-LogRecordSmtp -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "IMAP4Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "POP3Log" { #Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." $jobObject = Start-RSJob -Batch $batchJobId -Name "$($resultPreviousFile[0].LogFolder)\$($resultPreviousFile[0].LogFileName)" -FunctionsToImport Expand-LogRecordPopImap -Verbose:$false -ScriptBlock { Expand-LogRecordPopImap -InputObject $using:resultPreviousFile -sessionIdName $using:sessionIdName } } "MessageTrackingLog" { Write-PSFMessage -Level Host -Message "$($LogType) currently not supported." } Default { Write-PSFMessage -Level Warning -Message "Unknown LogType: $($LogType) | Probably developers mistake." } } $recordCount = $recordCount + $resultPreviousFile.count Write-PSFMessage -Level Verbose -Message "Start runspace job '$($jobObject.Name)' (ID:$($jobObject.ID)) for processing $($resultPreviousFile.count) record(s)" # waiting for completion of all runspaces Write-PSFMessage -Level Verbose -Message "Finished processing $($files.Count) file(s) with overall $($recordCount) record(s). Awaiting running runspaces to complete record processing." do { $jobs = Get-RSJob -Batch $batchJobId -Verbose:$false -ErrorAction SilentlyContinue | Sort-Object ID $jobsOpen = $jobs | Where-Object State -NotLike "Completed" $jobsCompleted = $jobs | Where-Object State -Like "Completed" [string]$_names = $jobsCompleted.Name | Split-Path -Leaf -ErrorAction SilentlyContinue if (-not $_names) { $_names = "" } Write-PSFMessage -Level VeryVerbose -Message "Awaiting runspaces to complete processing: $($jobsOpen.count)/$($jobs.count) ($([string]::Join(", ", $_names)))" Start-Sleep -Milliseconds 200 } while ($jobsOpen) # output remaining data in runspace Wait-JobCompleteWithOutput -Job $jobs Write-PSFMessage -Level Verbose -Message "All runspaces completed. Gathering results" $recordsProcessed = Receive-RSJob -Job $jobs -Verbose:$false foreach ($recordProcessed in $recordsProcessed) { [void]$recordProcessed.psobject.TypeNames.Remove('Selected.RSJob') $recordProcessed } Remove-RSJob -Batch $batchJobId -Verbose:$false $traceTimer.Stop() Write-PSFMessage -Level Significant -Message "Duration on parsing $($files.count) file(s) with $($recordCount) records: $($traceTimer.Elapsed)" } } <# This is an example configuration file By default, it is enough to have a single one of them, however if you have enough configuration settings to justify having multiple copies of it, feel totally free to split them into multiple files. #> <# # Example Configuration Set-PSFConfig -Module 'ExchangeLogs' -Name 'Example.Setting' -Value 10 -Initialize -Validation 'integer' -Handler { } -Description "Example configuration setting. Your module can then use the setting using 'Get-PSFConfigValue'" #> Set-PSFConfig -Module 'ExchangeLogs' -Name 'Import.DoDotSource' -Value $false -Initialize -Validation 'bool' -Description "Whether the module files should be dotsourced on import. By default, the files of this module are read as string value and invoked, which is faster but worse on debugging." Set-PSFConfig -Module 'ExchangeLogs' -Name 'Import.IndividualFiles' -Value $false -Initialize -Validation 'bool' -Description "Whether the module files should be imported individually. During the module build, all module code is compiled into few files, which are imported instead by default. Loading the compiled versions is faster, using the individual files is easier for debugging and testing out adjustments." Set-PSFConfig -Module 'ExchangeLogs' -Name 'SupportedLogTypes' -Value @( "SMTPReceiveProtocolLog", "SMTPSendProtocolLog", "IMAP4Log", "POP3Log", "MessageTrackingLog" ) -Initialize -Description "The name of the logfile types, specified in the header information of any exchange log file. This collection provides a list of supported formats to parsing for module ExchangeLogs." <# Stored scriptblocks are available in [PsfValidateScript()] attributes. This makes it easier to centrally provide the same scriptblock multiple times, without having to maintain it in separate locations. It also prevents lengthy validation scriptblocks from making your parameter block hard to read. Set-PSFScriptblock -Name 'ExchangeLogs.ScriptBlockName' -Scriptblock { } #> <# # Example: Register-PSFTeppScriptblock -Name "ExchangeLogs.alcohol" -ScriptBlock { 'Beer','Mead','Whiskey','Wine','Vodka','Rum (3y)', 'Rum (5y)', 'Rum (7y)' } #> <# # Example: Register-PSFTeppArgumentCompleter -Command Get-Alcohol -Parameter Type -Name ExchangeLogs.alcohol #> New-PSFLicense -Product 'ExchangeLogs' -Manufacturer 'andre' -ProductVersion $script:ModuleVersion -ProductType Module -Name MIT -Version "1.0.0.0" -Date (Get-Date "2020-06-13") -Text @" Copyright (c) 2020 Andreas Bellstedt Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. "@ #endregion Load compiled code |