Private/EventPattern.ps1
|
# How the events a Finding counts are spread in time. # # 72 lost connections in one half hour, 60 refusals at 10:01 and 14:01 every day, 11 drops # over five days and a failure every minute for ten hours are four different causes, and the # count says none of them. The Judges that count events per Subject say the spread beside # the count, and every one of them says it with this, so that the words and the thresholds # are the same wherever a Technician reads them. Judged, not drawn (ADR-0002): the Report and # the views only show what this said. function Get-EventPatternThreshold { <# .SYNOPSIS The thresholds of the event patterns: those a Check Definition names, and the defaults where it names none. Pure. .DESCRIPTION Parameters of the Check Definitions of the Kinds that say a pattern (ADR-0001), so that a Second Level engineer tunes them without a module release. The defaults are those the prototype settled on a real Run. .PARAMETER Parameters The Check Definition's parameters, or the hashtable this returned before: missing keys take their defaults either way. #> [CmdletBinding()] [OutputType([hashtable])] param([AllowNull()][hashtable]$Parameters) @{ PatternMinimumEvents = [int](Get-Parameter -Parameters $Parameters -Name 'PatternMinimumEvents' -Default 5) PatternMinimumDays = [double](Get-Parameter -Parameters $Parameters -Name 'PatternMinimumDays' -Default 5) PatternBurstGapMinutes = [double](Get-Parameter -Parameters $Parameters -Name 'PatternBurstGapMinutes' -Default 30) PatternBurstShare = [double](Get-Parameter -Parameters $Parameters -Name 'PatternBurstShare' -Default 0.8) PatternBurstMaxMinutes = [double](Get-Parameter -Parameters $Parameters -Name 'PatternBurstMaxMinutes' -Default 120) PatternDailyMinimumDays = [int](Get-Parameter -Parameters $Parameters -Name 'PatternDailyMinimumDays' -Default 5) PatternDailyShare = [double](Get-Parameter -Parameters $Parameters -Name 'PatternDailyShare' -Default 0.4) PatternDailyMaxTimes = [int](Get-Parameter -Parameters $Parameters -Name 'PatternDailyMaxTimes' -Default 3) PatternIntervalMinimumEvents = [int](Get-Parameter -Parameters $Parameters -Name 'PatternIntervalMinimumEvents' -Default 7) PatternIntervalShare = [double](Get-Parameter -Parameters $Parameters -Name 'PatternIntervalShare' -Default 0.8) PatternEdgeDays = [double](Get-Parameter -Parameters $Parameters -Name 'PatternEdgeDays' -Default 2) } } function New-EventPatternPeriod { <# .SYNOPSIS The period a Kind's events are judged in, as Get-EventPattern takes it: from when the logs cover it, the time of the Run, its days and the thresholds. Nothing where the data does not say when the period began. Pure. .DESCRIPTION A log overwrites its oldest records, so one that reaches back a week covers a week of a month: "erst seit" said of its first event would be the start of the log, and a daily time on 5 of 30 days would be on 5 of 7. The log that reaches back least limits it. .PARAMETER Data A Gatherer's data with Days, Since and LogCoverage, as the NetworkDisruption Kind describes them. .PARAMETER Parameters The Check Definition's parameters: its Days where the data has none, and the thresholds (Get-EventPatternThreshold). .PARAMETER Log The logs the events were read from. Without it, every log the data covers. #> [CmdletBinding()] [OutputType([psobject])] param( [AllowNull()]$Data, [AllowNull()][hashtable]$Parameters, [AllowNull()][AllowEmptyCollection()][string[]]$Log ) $since = ConvertTo-DataTime (Get-DataProperty $Data 'Since') if ($null -eq $since) { return } $days = Get-DataProperty $Data 'Days' if ($null -eq $days) { $days = Get-Parameter -Parameters $Parameters -Name 'Days' -Default 30 } $logs = @($Log | Where-Object { $_ }) $covered = $since foreach ($one in (Get-DataCollection $Data 'LogCoverage')) { if ($logs.Count -and $logs -notcontains "$(Get-DataProperty $one 'Log')") { continue } $oldest = ConvertTo-DataTime (Get-DataProperty $one 'Oldest') if ($null -ne $oldest -and $oldest -gt $covered) { $covered = $oldest } } [pscustomobject]@{ CoveredFrom = $covered RunAt = $since.AddDays([double]$days) Days = [int]$days Threshold = Get-EventPatternThreshold -Parameters $Parameters } } function Get-EventPattern { <# .SYNOPSIS How the events of one Subject and one kind of event are spread in time, as tokens and as the sentence a list item says behind its span: "Ausbruch: 102 von 102, 14.10. 22:12 bis 22:38", "täglich gegen 10:01 und 14:01, an 30 von 30 Tagen", "jede Minute, 14.10. 22:09 bis 15.10. 08:24, erst seit 14.10.", "verstreut". Pure. .DESCRIPTION Tokens, in the order they are said: burst, daily, interval, working-hours, onset, end, scattered. Scattered is what is left when there is no burst, no daily time and no interval: the spread was looked at and is irregular. Nothing is said rather than guessed: 'few' where there are too few events to call anything a trend, with no sentence, and 'short' where the logs cover too few days, with a sentence that says why - a two-day log is not to be read as a month. One thing is said once. A burst is not also an interval: a hundred events in half an hour are a minute apart because they are a burst. Nor does it have an onset or an end: its window says both. An interval is not also daily: events every 1.6 hours come back at the same times every day because they are an interval. Times are local. A job at 10:01 stays at 10:01 across the change of the clock. A job scheduled in UTC moves by an hour, out of the quarters beside its own, and is said as the two times it was run at. .PARAMETER Time When the events happened. Unsorted; what is no time is left out. .PARAMETER CoveredFrom From when the logs of these events cover the period: the later of the period's start and how far back the log reaches. An event at the start of a log is not the start of a fault, so an onset is only said after this. .PARAMETER RunAt When the period ends: the time of the Run. .PARAMETER Days The period's length in days. Tells a log that reaches back too little from a period that is short in itself. .PARAMETER Threshold As Get-EventPatternThreshold returns it; what it leaves out takes the default. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [AllowNull()][AllowEmptyCollection()][object[]]$Time, [Parameter(Mandatory)][datetime]$CoveredFrom, [Parameter(Mandatory)][datetime]$RunAt, [Parameter(Mandatory)][int]$Days, [AllowNull()][hashtable]$Threshold ) $limit = Get-EventPatternThreshold -Parameters $Threshold $list = [System.Collections.Generic.List[datetime]]::new() foreach ($value in @($Time)) { if ($null -eq $value) { continue } $at = ConvertTo-DataTime $value if ($at -is [datetime]) { $list.Add($at) } } if ($list.Count -lt $limit.PatternMinimumEvents) { return [pscustomobject]@{ Tokens = [string[]]@('few'); Text = '' } } # Covered days: from the later of the period's start and the log's reach, to the Run. $covered = ($RunAt - $CoveredFrom).TotalDays if ($covered -lt $limit.PatternMinimumDays) { $reach = [math]::Max(1, [int][math]::Round($covered)) $said = if ($reach -eq 1) { Get-Text 'Value.EventPattern.Days.One' } else { Get-Text 'Value.EventPattern.Days.Many' $reach } # The period itself is short where the log reaches back to its start; otherwise the # log is what is short. A minute either way is the same start. $text = if ($CoveredFrom -gt $RunAt.AddDays(-$Days).AddMinutes(1)) { Get-Text 'Value.EventPattern.Short.Log' $said } else { Get-Text 'Value.EventPattern.Short.Period' $said } return [pscustomobject]@{ Tokens = [string[]]@('short'); Text = $text } } $sorted = $list.ToArray() [array]::Sort($sorted) $count = $sorted.Count $first = $sorted[0] $last = $sorted[$count - 1] # Burst: events no further apart than the gap form clusters; the largest holds most of # them in a short window. $bestFrom = 0; $bestTo = 0; $from = 0 for ($i = 1; $i -le $count; $i++) { if ($i -eq $count -or ($sorted[$i] - $sorted[$i - 1]).TotalMinutes -gt $limit.PatternBurstGapMinutes) { if (($i - $from) -gt ($bestTo - $bestFrom + 1)) { $bestFrom = $from; $bestTo = $i - 1 } $from = $i } } $burstCount = $bestTo - $bestFrom + 1 $burst = $burstCount -ge ($limit.PatternBurstShare * $count) -and ($sorted[$bestTo] - $sorted[$bestFrom]).TotalMinutes -le $limit.PatternBurstMaxMinutes # Interval, on the events reduced to one per minute. The median gap and not the mean: six # longer pauses among 598 gaps of a minute moved the mean and hid a retry every minute. $interval = $false; $median = 0.0 if (-not $burst) { $minutes = [System.Collections.Generic.List[long]]::new() $previous = [long]-1 foreach ($at in $sorted) { $minute = [long][math]::Floor($at.Ticks / [timespan]::TicksPerMinute) if ($minute -ne $previous) { $minutes.Add($minute); $previous = $minute } } if ($minutes.Count -ge $limit.PatternIntervalMinimumEvents) { $gaps = [double[]]@(for ($i = 1; $i -lt $minutes.Count; $i++) { $minutes[$i] - $minutes[$i - 1] }) $byLength = [double[]]$gaps.Clone() [array]::Sort($byLength) $middle = [int][math]::Floor($byLength.Count / 2) $median = if ($byLength.Count % 2) { $byLength[$middle] } else { ($byLength[$middle - 1] + $byLength[$middle]) / 2 } $within = [math]::Max(0.5, $median * 0.25) $alike = @($gaps | Where-Object { [math]::Abs($_ - $median) -le $within }).Count $interval = $median -lt 1440 -and ($alike / $gaps.Count) -ge $limit.PatternIntervalShare } } # Daily: quarter hours of the day, each with the one either side, round across midnight. # The quarter with events on the most days is a daily time when those are enough days and # enough of the days with events left; then again without its events, up to the most # times there are. A job at 10:01 lies in the quarters either side of its own on every # day, and a job an hour off lies outside them. # An interval of a few hours comes back at the same times every day because it is an # interval, and is said as one. One the day holds no more often than there are daily # times - 10:01 and 22:01 - is a job of the day, and daily says when. $daily = @() if (-not $interval -or $median -ge 1440 / [math]::Max(1, $limit.PatternDailyMaxTimes)) { $remaining = @(foreach ($at in $sorted) { $ofDay = $at.Hour * 60 + $at.Minute [pscustomobject]@{ Minute = $ofDay; Quarter = [int][math]::Floor($ofDay / 15); Day = $at.ToString('yyyyMMdd') } }) for ($round = 0; $round -lt $limit.PatternDailyMaxTimes -and $remaining.Count; $round++) { $daysIn = @{} foreach ($one in $remaining) { if (-not $daysIn.ContainsKey($one.Quarter)) { $daysIn[$one.Quarter] = @{} } $daysIn[$one.Quarter][$one.Day] = $true } $withEvents = @($remaining | ForEach-Object { $_.Day } | Select-Object -Unique).Count $best = -1; $bestDays = 0; $bestOwn = 0 foreach ($quarter in 0..95) { $union = @{} foreach ($near in @((($quarter + 95) % 96), $quarter, (($quarter + 1) % 96))) { if ($daysIn.ContainsKey($near)) { foreach ($day in $daysIn[$near].Keys) { $union[$day] = $true } } } $own = if ($daysIn.ContainsKey($quarter)) { $daysIn[$quarter].Count } else { 0 } if ($union.Count -gt $bestDays -or ($union.Count -eq $bestDays -and $own -gt $bestOwn)) { $best = $quarter; $bestDays = $union.Count; $bestOwn = $own } } if ($best -lt 0 -or $bestDays -lt $limit.PatternDailyMinimumDays -or $bestDays -lt ($limit.PatternDailyShare * $withEvents)) { break } $window = @((($best + 95) % 96), $best, (($best + 1) % 96)) $members = @($remaining | Where-Object { $window -contains $_.Quarter }) $remaining = @($remaining | Where-Object { $window -notcontains $_.Quarter }) # The median minute, measured from the middle of its quarter so that a time just # before midnight and one just after are minutes apart and not a day. $centre = $best * 15 + 7.5 $offsets = [double[]]@($members | ForEach-Object { ((($_.Minute - $centre + 720) % 1440) + 1440) % 1440 - 720 }) [array]::Sort($offsets) $said = [int]((([math]::Round($centre + $offsets[[int][math]::Floor(($offsets.Count - 1) / 2)]) % 1440) + 1440) % 1440) $daily += [pscustomobject]@{ Minute = $said; Days = $bestDays } } # One thing is said once: the times of the day, not the gap between them as well. if ($daily.Count) { $interval = $false } } $tokens = @() $parts = @() $span = Format-FindingSpan -First $first -Last $last if ($burst) { $tokens += 'burst' $parts += Get-Text 'Value.EventPattern.Burst' $burstCount $count (Format-FindingSpan -First $sorted[$bestFrom] -Last $sorted[$bestTo]) } if ($daily.Count) { $tokens += 'daily' $clock = @($daily | Sort-Object Minute | ForEach-Object { '{0:00}:{1:00}' -f [math]::Floor($_.Minute / 60), ($_.Minute % 60) }) $times = if ($clock.Count -eq 1) { $clock[0] } else { Get-Text 'Value.EventPattern.And' (($clock[0..($clock.Count - 2)]) -join ', ') $clock[-1] } # Of the covered days, never fewer than the days it was seen on. $of = [math]::Max($daily[0].Days, [int][math]::Round($covered)) $parts += Get-Text 'Value.EventPattern.Daily' $times $parts += Get-Text 'Value.EventPattern.DailyDays' $daily[0].Days $of } if ($interval) { $tokens += 'interval' $every = [int][math]::Round($median) $parts += if ($median -le 10) { # A few minutes or less is a machine retrying, said as the loop it is. if ($every -le 1) { Get-Text 'Value.EventPattern.Interval.Minute' $span } else { Get-Text 'Value.EventPattern.Interval.Minutes' $every $span } } elseif ($median -lt 60) { Get-Text 'Value.EventPattern.Interval.RegularMinutes' $every $span } else { $hours = $median / 60 if ([math]::Round($hours, 1) -eq 1) { Get-Text 'Value.EventPattern.Interval.Hour' $span } else { Get-Text 'Value.EventPattern.Interval.Hours' $hours.ToString('0.#', [cultureinfo]::GetCultureInfo('de-DE')) $span } } } # Scattered is said with the shapes, first of what is said beside it: "verstreut, erst # seit 10.10." reads as the spread and then when. if (-not $burst -and -not $daily.Count -and -not $interval) { $tokens += 'scattered' $parts += Get-Text 'Value.EventPattern.Scattered' } # Working hours: on several days, and none on a weekend or outside the day's work. $onDays = @($sorted | ForEach-Object { $_.Date } | Select-Object -Unique).Count $outside = @($sorted | Where-Object { $_.DayOfWeek -eq [DayOfWeek]::Saturday -or $_.DayOfWeek -eq [DayOfWeek]::Sunday -or $_.Hour -lt 7 -or $_.Hour -ge 19 }).Count if ($onDays -ge 3 -and -not $outside) { $tokens += 'working-hours' $parts += Get-Text 'Value.EventPattern.WorkingHours' } if (-not $burst) { if ($first -ge $CoveredFrom.AddDays($limit.PatternEdgeDays)) { $tokens += 'onset' $parts += Get-Text 'Value.EventPattern.Onset' $first } if ($last -le $RunAt.AddDays(-$limit.PatternEdgeDays)) { $tokens += 'end' $parts += Get-Text 'Value.EventPattern.End' $last } } [pscustomobject]@{ Tokens = [string[]]$tokens; Text = ($parts -join ', ') } } function Get-EventPatternByKind { <# .SYNOPSIS How the events of one Subject are spread in time, per kind of event, as its list item and its Subject part say it: "verstreut" where they are of one kind, "SMB-Verbindung getrennt: verstreut / <the timeouts>: Ausbruch: ..." where they are of several. Nothing where the period of a kind is not known. Pure. .DESCRIPTION Per kind, because a burst of timeouts and a month of scattered drops of one Server are two things, and blended they read as "verstreut". Events of one kind are said as Get-EventPattern words them. Events of several are each said with the kind in front, the kind with the most events first, and a kind too few to say anything of is left out: the kind stays in front of what is said, because several were counted. That the logs reach back too few days is said once, where that is all it says and it is the same for every kind. Each kind is judged in the period its own logs cover: a log that reaches back a week limits the pattern of its events, not of those another log holds. Tokens are those of every kind, each once, without 'few' and 'short': those say that nothing was said. .PARAMETER Row The events of one Subject, each with its kind and its time. .PARAMETER KindProperty The property that holds an event's kind. .PARAMETER TimeProperty The property that holds when it happened. .PARAMETER KindText A kind in words, as the Kind's Section names it: given the kind, returns the words. .PARAMETER Period The period a kind's events are judged in: given the kind, returns what New-EventPatternPeriod returns for the logs it is read from, or nothing where the data does not say when the period began. .PARAMETER Labelled Says the kind in front even where the events are of one: for a Finding whose other Subjects have several, so that every one of its Subjects says it alike. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [AllowNull()][AllowEmptyCollection()]$Row, [string]$KindProperty = 'Kind', [string]$TimeProperty = 'Time', [Parameter(Mandatory)][scriptblock]$KindText, [Parameter(Mandatory)][scriptblock]$Period, [switch]$Labelled ) $groups = @(@($Row | Where-Object { $_ }) | Group-Object { "$(Get-DataProperty $_ $KindProperty)" } | Sort-Object @{ Expression = 'Count'; Descending = $true }, Name) $spans = @(foreach ($group in $groups) { & $Period $group.Name }) if (@($spans | Where-Object { $null -ne $_ }).Count -lt $groups.Count) { return } $each = @(for ($i = 0; $i -lt $groups.Count; $i++) { $span = $spans[$i] $times = @($groups[$i].Group | ForEach-Object { ConvertTo-DataTime (Get-DataProperty $_ $TimeProperty) } | Where-Object { $null -ne $_ }) [pscustomobject]@{ Kind = $groups[$i].Name Pattern = Get-EventPattern -Time $times -CoveredFrom $span.CoveredFrom -RunAt $span.RunAt -Days $span.Days -Threshold $span.Threshold } }) $tokens = @($each | ForEach-Object { $_.Pattern.Tokens } | Where-Object { $_ -notin 'few', 'short' } | Select-Object -Unique) $spoken = @($each | Where-Object { "$($_.Pattern.Text)" }) $why = @($spoken | ForEach-Object { "$($_.Pattern.Text)" } | Select-Object -Unique) $text = '' if (($groups.Count -eq 1 -and -not $Labelled) -or (-not $tokens.Count -and $why.Count -eq 1)) { if ($spoken.Count) { $text = "$($spoken[0].Pattern.Text)" } } else { foreach ($one in $spoken) { $part = Get-Text 'Value.EventPattern.OfKind' "$(& $KindText $one.Kind)" "$($one.Pattern.Text)" $text = if ($text) { Get-Text 'Value.EventPattern.Between' $text $part } else { $part } } } [pscustomobject]@{ Tokens = [string[]]$tokens; Text = $text } } function Format-EventPatternItem { <# .SYNOPSIS A list item with the pattern of its events behind its span: "FS01 102x, 14.10. 22:12 bis 22:38 - Ausbruch: 102 von 102, 14.10. 22:12 bis 22:38". The item as it was where the pattern says nothing. Pure. .DESCRIPTION Its own part of the item, so that the Report shows it as a column beside the count and the span (Split-ReportTally). .PARAMETER Item The list item as it is written without it: what, how often and from when to when. .PARAMETER Pattern What Get-EventPattern returned for the events the item counts. #> [CmdletBinding()] [OutputType([string])] param( [AllowEmptyString()][string]$Item, [AllowNull()]$Pattern ) if ($null -eq $Pattern -or -not "$($Pattern.Text)") { return $Item } Get-Text 'Value.EventPattern.Item' $Item "$($Pattern.Text)" } |