Payload/scripts/kaden/utils/rank-harness-file-times.ps1
|
<# .SYNOPSIS Rank harness test files by duration and example count. .DESCRIPTION Reads Pester file-summary lines from harness-run-slice-tests sensor JSON and ranks each *.Tests.ps1 file. Average individual test time is file seconds divided by the example count. The highest values are listed first. Pass several sensors from the same suite to average them. Pass an older sensor with -BaselineSensorPath to print how the totals moved. A Summary section is always last so a later run can be compared by eye. This reads sensor logs. It does not start the suite. .PARAMETER SensorPath One or more sensor JSON files to average. Pass several files as one comma-separated value. pwsh -File does not repeat a parameter and does not split that comma itself, so this script splits it. Paths must not contain commas. When omitted, the newest full-suite sensors under the sensor root are selected. .PARAMETER BaselineSensorPath Older sensor JSON files, also as one comma-separated value. Their file times are averaged and compared with the current ranking in the summary. .PARAMETER Feature Limit automatic discovery to docs/harness/features/<Feature>/sensors. .PARAMETER SensorRoot Folder that contains feature directories. Defaults to docs/harness/features under the repository root. .PARAMETER Runs How many full-suite sensors to average when SensorPath is omitted. Default: 1. .PARAMETER ScanLimit How many newest sensor files to open while looking for full-suite logs. Default: 200. .PARAMETER MinFileLines A discovered sensor counts as a full suite when it has at least this many passing file-summary lines. Default: 15. Explicit SensorPath values are not held to this minimum. .PARAMETER MinFileSeconds Omit files whose mean file time is below this many seconds from the table. They remain in the summary as held out. Default: 0. .PARAMETER Top How many ranked rows to print. 0 prints every row that passed MinFileSeconds. Default: 0. .PARAMETER Sort AverageExample ranks by file seconds / example count. FileSeconds ranks by the whole file time. Default: AverageExample. .PARAMETER ShowHelp Show comment-based help and return. .EXAMPLE pwsh -NoProfile -File .\scripts\kaden\utils\rank-harness-file-times.ps1 -Runs 3 -Feature harness-event-journal .EXAMPLE pwsh -NoProfile -File .\scripts\kaden\utils\rank-harness-file-times.ps1 -SensorPath .\docs\harness\features\harness-event-journal\sensors\harness-run-slice-tests-20261014T000839Z.json,.\docs\harness\features\harness-event-journal\sensors\harness-run-slice-tests-20261014T020940Z.json,.\docs\harness\features\harness-event-journal\sensors\harness-run-slice-tests-20261014T030750Z.json .EXAMPLE pwsh -NoProfile -File .\scripts\kaden\utils\rank-harness-file-times.ps1 -SensorPath .\docs\harness\features\harness-event-journal\sensors\harness-run-slice-tests-20261014T030750Z.json -BaselineSensorPath .\docs\harness\features\harness-event-journal\sensors\harness-run-slice-tests-20261014T000839Z.json .EXAMPLE pwsh -NoProfile -File .\scripts\kaden\utils\rank-harness-file-times.ps1 -h #> #Requires -Version 5.1 [CmdletBinding(DefaultParameterSetName = "Run")] param( [Parameter(ParameterSetName = "Run")] [string[]]$SensorPath = @(), [Parameter(ParameterSetName = "Run")] [string[]]$BaselineSensorPath = @(), [Parameter(ParameterSetName = "Run")] [string]$Feature = "", [Parameter(ParameterSetName = "Run")] [string]$SensorRoot = "", [Parameter(ParameterSetName = "Run")] [ValidateRange(1, 20)] [int]$Runs = 1, [Parameter(ParameterSetName = "Run")] [ValidateRange(1, 2000)] [int]$ScanLimit = 200, [Parameter(ParameterSetName = "Run")] [ValidateRange(1, 500)] [int]$MinFileLines = 15, [Parameter(ParameterSetName = "Run")] [ValidateRange(0, 86400)] [double]$MinFileSeconds = 0, [Parameter(ParameterSetName = "Run")] [ValidateRange(0, 500)] [int]$Top = 0, [Parameter(ParameterSetName = "Run")] [ValidateSet("AverageExample", "FileSeconds")] [string]$Sort = "AverageExample", [Parameter(ParameterSetName = "Help")] [Alias("h", "help")] [switch]$ShowHelp ) Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" # --------------------- helpers --------------------- function Resolve-RankRepoRoot { $current = $PSScriptRoot while ($true) { $marker = Join-Path $current "package.json" if (Test-Path -LiteralPath $marker) { return $current } $parent = Split-Path $current -Parent if ([string]::IsNullOrWhiteSpace($parent) -or $parent -eq $current) { throw "Could not locate repository root (package.json) from this script." } $current = $parent } } function Expand-RankPathList { param([string[]]$Path) $expanded = @() foreach ($item in @($Path)) { if ([string]::IsNullOrWhiteSpace($item)) { continue } foreach ($piece in ($item -split ",")) { $trimmed = $piece.Trim() if (-not [string]::IsNullOrWhiteSpace($trimmed)) { $expanded += $trimmed } } } return $expanded } function Resolve-RankInputPath { param( [Parameter(Mandatory = $true)][string]$InputPath, [Parameter(Mandatory = $true)][string]$RepoRoot ) if (Test-Path -LiteralPath $InputPath) { return (Resolve-Path -LiteralPath $InputPath).Path } if (-not [System.IO.Path]::IsPathRooted($InputPath)) { $candidate = Join-Path $RepoRoot $InputPath if (Test-Path -LiteralPath $candidate) { return (Resolve-Path -LiteralPath $candidate).Path } } throw "Sensor file not found: $InputPath. Get-Help `"$PSCommandPath`" -Parameter SensorPath" } function Read-RankSensor { param( [Parameter(Mandatory = $true)][string]$Path ) $sensor = Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json $text = "" foreach ($field in @("stdout_tail", "stdout")) { if ($sensor.PSObject.Properties.Name -contains $field) { $piece = [string]$sensor.$field if (-not [string]::IsNullOrWhiteSpace($piece)) { if ($text.Length -gt 0) { $text += "`n" } $text += $piece } } } $started = "" $finished = "" $verdict = "" $name = "" if ($sensor.PSObject.Properties.Name -contains "started_at") { $started = Format-RankInstant -Value $sensor.started_at } if ($sensor.PSObject.Properties.Name -contains "completed_at") { $finished = Format-RankInstant -Value $sensor.completed_at } if ($sensor.PSObject.Properties.Name -contains "verdict") { $verdict = [string]$sensor.verdict } if ($sensor.PSObject.Properties.Name -contains "name") { $name = [string]$sensor.name } return [pscustomobject]@{ Path = $Path Name = $name Started = $started Finished = $finished Verdict = $verdict Text = $text } } function Get-RankFileDuration { param( [Parameter(Mandatory = $true)][AllowEmptyString()][string]$Text ) $pattern = '(?m)^\[(\+|-)\]\s+\S*?([A-Za-z0-9_.-]+\.Tests\.ps1)\s+([0-9]+(?:\.[0-9]+)?)(ms|s)\s+\((\d+)\s+tests?\)\s*$' $seen = @{} $rows = @() $expression = New-Object System.Text.RegularExpressions.Regex $pattern foreach ($match in @($expression.Matches($Text))) { $fileName = [string]$match.Groups[2].Value if ($seen.ContainsKey($fileName)) { continue } $seen[$fileName] = $true $magnitude = [double]$match.Groups[3].Value $seconds = $magnitude if ([string]$match.Groups[4].Value -eq "ms") { $seconds = $magnitude / 1000.0 } $passed = ([string]$match.Groups[1].Value -eq "+") $rows += [pscustomobject]@{ FileName = $fileName Seconds = $seconds Examples = [int]$match.Groups[5].Value Passed = $passed } } return $rows } function Get-RankSensorStamp { param( [Parameter(Mandatory = $true)][string]$Path ) $name = [System.IO.Path]::GetFileName($Path) $match = [System.Text.RegularExpressions.Regex]::Match($name, '(\d{8}T\d{6}Z)') if ($match.Success) { return $match.Groups[1].Value } return $name } function Get-RankRollup { param( [Parameter(Mandatory = $true)][object[]]$Sensor ) $byFile = @{} $failed = @() $sensorRows = @() $ordered = @($Sensor | Sort-Object @{ Expression = { Get-RankSensorStamp -Path $_.Path } }, @{ Expression = { $_.Started } }) foreach ($one in $ordered) { $durations = @(Get-RankFileDuration -Text ([string]$one.Text)) $passing = @($durations | Where-Object { $_.Passed }) $sensorRows += [pscustomobject]@{ Path = $one.Path Started = $one.Started Finished = $one.Finished Verdict = $one.Verdict FileLines = $passing.Count } foreach ($row in $durations) { if (-not $row.Passed) { $failed += [pscustomobject]@{ FileName = $row.FileName Seconds = $row.Seconds Examples = $row.Examples Sensor = [System.IO.Path]::GetFileName($one.Path) } continue } if (-not $byFile.ContainsKey($row.FileName)) { $byFile[$row.FileName] = [pscustomobject]@{ FileName = $row.FileName Samples = @() Examples = $row.Examples ExampleCountMismatch = $false } } $bucket = $byFile[$row.FileName] $bucket.Samples = @($bucket.Samples) + [double]$row.Seconds if ($bucket.Examples -ne $row.Examples) { $bucket.ExampleCountMismatch = $true $bucket.Examples = $row.Examples } } } $files = @() foreach ($key in @($byFile.Keys)) { $bucket = $byFile[$key] $sum = 0.0 foreach ($value in @($bucket.Samples)) { $sum += [double]$value } $sampleCount = @($bucket.Samples).Count $mean = $sum / $sampleCount $perExample = 0.0 if ($bucket.Examples -gt 0) { $perExample = $mean / $bucket.Examples } $files += [pscustomobject]@{ FileName = $bucket.FileName MeanSeconds = $mean Examples = $bucket.Examples Runs = $sampleCount SecondsPerExample = $perExample ExampleCountMismatch = [bool]$bucket.ExampleCountMismatch } } return [pscustomobject]@{ Files = @($files) Failed = @($failed) Sensors = @($sensorRows) } } function Get-RankDiscoveredSensor { param( [Parameter(Mandatory = $true)][string]$Root, [string]$FeatureName, [int]$RunCount, [int]$OpenLimit, [int]$MinimumLines ) $search = $Root if (-not [string]::IsNullOrWhiteSpace($FeatureName)) { $search = Join-Path (Join-Path $Root $FeatureName) "sensors" } if (-not (Test-Path -LiteralPath $search)) { throw "Sensor root not found: $search. Get-Help `"$PSCommandPath`" -Parameter SensorRoot" } $candidates = @(Get-ChildItem -LiteralPath $search -Recurse -File -Filter "harness-run-slice-tests-*.json" | Sort-Object @{ Expression = { Get-RankSensorStamp -Path $_.Name } } -Descending) $chosen = @() $opened = 0 foreach ($file in $candidates) { if ($opened -ge $OpenLimit -or @($chosen).Count -ge $RunCount) { break } $opened++ $sensor = Read-RankSensor -Path $file.FullName $passing = @((Get-RankFileDuration -Text $sensor.Text) | Where-Object { $_.Passed }) if ($passing.Count -ge $MinimumLines) { $chosen += $sensor } } return [pscustomobject]@{ Sensors = @($chosen) Opened = $opened Seen = $candidates.Count } } function Format-RankDuration { param([double]$Value) return [string]::Format([System.Globalization.CultureInfo]::InvariantCulture, "{0:0.00}", $Value) } function Format-RankInstant { param($Value) if ($null -eq $Value) { return "" } if ($Value -is [datetime]) { $instant = [datetime]$Value if ($instant.Kind -eq [DateTimeKind]::Utc) { return $instant.ToString("yyyy-MM-ddTHH:mm:ss'Z'") } return $instant.ToString("yyyy-MM-ddTHH:mm:ss") } return [string]$Value } function Get-RankOrderedFile { param( [Parameter(Mandatory = $true)][object[]]$File, [Parameter(Mandatory = $true)][string]$SortName, [double]$MinimumSeconds ) $visible = @($File | Where-Object { $_.MeanSeconds -ge $MinimumSeconds -or $MinimumSeconds -eq 0 }) if ($MinimumSeconds -gt 0) { $visible = @($File | Where-Object { $_.MeanSeconds + 0.0000001 -ge $MinimumSeconds }) } if ($SortName -eq "FileSeconds") { return @($visible | Sort-Object @{ Expression = "MeanSeconds"; Descending = $true }, @{ Expression = "FileName" }) } return @($visible | Sort-Object @{ Expression = "SecondsPerExample"; Descending = $true }, @{ Expression = "FileName" }) } function Show-RankTable { param( [Parameter(Mandatory = $true)][object[]]$Row ) $header = "{0,4} {1,10} {2,8} {3,9} {4,4} {5}" -f "rank", "s/example", "file_s", "examples", "runs", "file" Write-CliColor -Message $header -Literal Write-CliColor -Message ("-" * $header.Length) -Literal $rank = 0 foreach ($rowItem in $Row) { $rank++ $line = "{0,4} {1,10} {2,8} {3,9} {4,4} {5}" -f ` $rank, ` (Format-RankDuration -Value $rowItem.SecondsPerExample), ` (Format-RankDuration -Value $rowItem.MeanSeconds), ` $rowItem.Examples, ` $rowItem.Runs, ` $rowItem.FileName Write-CliColor -Message $line -Literal } } function Show-RankSensorList { param( [Parameter(Mandatory = $true)][string]$Title, [Parameter(Mandatory = $true)][object[]]$Sensor ) Write-CliColor -Message $Title -Literal foreach ($one in $Sensor) { $leaf = [System.IO.Path]::GetFileName($one.Path) $detail = "$leaf started=$($one.Started) verdict=$($one.Verdict) file_lines=$($one.FileLines)" Write-CliColor -Message (" " + $detail) -Literal } } function Get-RankSuiteTotal { param( [Parameter(Mandatory = $true)][object[]]$File ) $examples = 0 $seconds = 0.0 $mismatch = 0 foreach ($one in $File) { $examples += [int]$one.Examples $seconds += [double]$one.MeanSeconds if ($one.ExampleCountMismatch) { $mismatch++ } } $perExample = 0.0 if ($examples -gt 0) { $perExample = $seconds / $examples } return [pscustomobject]@{ Files = @($File).Count Examples = $examples FileSeconds = $seconds PerExample = $perExample Mismatch = $mismatch } } function Show-RankSummary { param( [Parameter(Mandatory = $true)][object]$Current, [object]$Baseline, [Parameter(Mandatory = $true)][object[]]$Shown, [Parameter(Mandatory = $true)][int]$HeldOut, [Parameter(Mandatory = $true)][string]$SortName, [int]$Opened = 0, [int]$Seen = 0 ) $totals = Get-RankSuiteTotal -File $Current.Files Write-CliSection -Title "Summary" if ($Opened -gt 0) { Write-CliKeyValue -Label "sensors opened" -Value "$Opened of $Seen newest" -LabelWidth 22 } Write-CliKeyValue -Label "sensors used" -Value ([string]$Current.Sensors.Count) -LabelWidth 22 Write-CliKeyValue -Label "files ranked" -Value ([string]$totals.Files) -LabelWidth 22 Write-CliKeyValue -Label "table rows" -Value ([string]@($Shown).Count) -LabelWidth 22 Write-CliKeyValue -Label "examples" -Value ([string]$totals.Examples) -LabelWidth 22 Write-CliKeyValue -Label "summed file seconds" -Value (Format-RankDuration -Value $totals.FileSeconds) -LabelWidth 22 Write-CliKeyValue -Label "suite s/example" -Value (Format-RankDuration -Value $totals.PerExample) -LabelWidth 22 Write-CliKeyValue -Label "held out" -Value ([string]$HeldOut) -LabelWidth 22 Write-CliKeyValue -Label "failed file lines" -Value ([string]@($Current.Failed).Count) -LabelWidth 22 if ($totals.Mismatch -gt 0) { $names = (@($Current.Files | Where-Object { $_.ExampleCountMismatch } | ForEach-Object { $_.FileName }) -join ", ") Write-CliKeyValue -Label "count mismatches" -Value ("{0} ({1})" -f $totals.Mismatch, $names) -LabelWidth 22 } if ($SortName -eq "FileSeconds") { Write-CliColor -Message " Order is whole-file seconds, highest first." -Literal } else { Write-CliColor -Message " Order is file seconds / example count, highest first." -Literal } $slowest = @(Get-RankOrderedFile -File $Current.Files -SortName "AverageExample" -MinimumSeconds 0 | Select-Object -First 5) Write-CliColor -Message " Slowest per example:" -Literal foreach ($one in $slowest) { $line = " {0} {1} s/example ({2}s, {3} examples)" -f ` $one.FileName, ` (Format-RankDuration -Value $one.SecondsPerExample), ` (Format-RankDuration -Value $one.MeanSeconds), ` $one.Examples Write-CliColor -Message $line -Literal } $largest = @(Get-RankOrderedFile -File $Current.Files -SortName "FileSeconds" -MinimumSeconds 0 | Select-Object -First 5) Write-CliColor -Message " Largest files:" -Literal foreach ($one in $largest) { $line = " {0} {1}s ({2} s/example, {3} examples)" -f ` $one.FileName, ` (Format-RankDuration -Value $one.MeanSeconds), ` (Format-RankDuration -Value $one.SecondsPerExample), ` $one.Examples Write-CliColor -Message $line -Literal } if (@($Current.Failed).Count -gt 0) { Write-CliColor -Message " Failed file lines were left out of the rank:" -Literal foreach ($one in @($Current.Failed | Select-Object -First 10)) { $line = " {0} {1}s ({2} examples) {3}" -f ` $one.FileName, ` (Format-RankDuration -Value $one.Seconds), ` $one.Examples, ` $one.Sensor Write-CliColor -Message $line -Literal } } if ($null -ne $Baseline) { $before = Get-RankSuiteTotal -File $Baseline.Files $saved = $before.FileSeconds - $totals.FileSeconds Write-CliColor -Message " Baseline comparison:" -Literal $compare = " summed file seconds {0} -> {1} (delta {2})" -f ` (Format-RankDuration -Value $before.FileSeconds), ` (Format-RankDuration -Value $totals.FileSeconds), ` (Format-RankDuration -Value $saved) Write-CliColor -Message $compare -Literal $rate = " suite s/example {0} -> {1}" -f ` (Format-RankDuration -Value $before.PerExample), ` (Format-RankDuration -Value $totals.PerExample) Write-CliColor -Message $rate -Literal $beforeMap = @{} foreach ($one in @($Baseline.Files)) { $beforeMap[$one.FileName] = $one } $movers = @() $added = 0 foreach ($one in @($Current.Files)) { if (-not $beforeMap.ContainsKey($one.FileName)) { $added++ continue } $previous = $beforeMap[$one.FileName] $movers += [pscustomobject]@{ FileName = $one.FileName Before = [double]$previous.MeanSeconds After = [double]$one.MeanSeconds Delta = [double]$previous.MeanSeconds - [double]$one.MeanSeconds } } $removed = 0 $currentNames = @{} foreach ($one in @($Current.Files)) { $currentNames[$one.FileName] = $true } foreach ($one in @($Baseline.Files)) { if (-not $currentNames.ContainsKey($one.FileName)) { $removed++ } } Write-CliKeyValue -Label "files only now" -Value ([string]$added) -LabelWidth 22 Write-CliKeyValue -Label "files only baseline" -Value ([string]$removed) -LabelWidth 22 $topDrops = @($movers | Sort-Object @{ Expression = "Delta"; Descending = $true } | Select-Object -First 5) Write-CliColor -Message " Largest file-second drops:" -Literal foreach ($one in $topDrops) { $line = " {0} {1}s -> {2}s (delta {3})" -f ` $one.FileName, ` (Format-RankDuration -Value $one.Before), ` (Format-RankDuration -Value $one.After), ` (Format-RankDuration -Value $one.Delta) Write-CliColor -Message $line -Literal } } Write-CliColor -Message (" TOTAL files={0} examples={1} file_seconds={2} suite_s_per_example={3}" -f ` $totals.Files, ` $totals.Examples, ` (Format-RankDuration -Value $totals.FileSeconds), ` (Format-RankDuration -Value $totals.PerExample)) -Literal } # --------------------- run --------------------- $cliKitManifest = Join-Path $PSScriptRoot "PsCliUiKit\PsCliUiKit.psd1" if (-not (Test-Path -LiteralPath $cliKitManifest)) { throw "PsCliUiKit manifest not found: $cliKitManifest" } Import-Module -Name $cliKitManifest -Force try { if ($ShowHelp) { (Get-Help -Name $PSCommandPath -Full | Out-String -Width 100) | Out-Host return } $repoRoot = Resolve-RankRepoRoot $discoveredNote = $null $currentSensors = @() $sensorPaths = @(Expand-RankPathList -Path $SensorPath) if ($sensorPaths.Count -gt 0) { foreach ($inputPath in $sensorPaths) { $fullPath = Resolve-RankInputPath -InputPath $inputPath -RepoRoot $repoRoot $currentSensors += Read-RankSensor -Path $fullPath } } else { $root = $SensorRoot if ([string]::IsNullOrWhiteSpace($root)) { $root = Join-Path $repoRoot "docs\harness\features" } elseif (-not [System.IO.Path]::IsPathRooted($root)) { $root = Join-Path $repoRoot $root } $discovered = Get-RankDiscoveredSensor -Root $root -FeatureName $Feature -RunCount $Runs -OpenLimit $ScanLimit -MinimumLines $MinFileLines $currentSensors = @($discovered.Sensors) $discoveredNote = $discovered if ($currentSensors.Count -eq 0) { throw "No full-suite sensor found under $root. Pass -SensorPath, or lower -MinFileLines. Get-Help `"$PSCommandPath`" -Parameter SensorPath" } } $current = Get-RankRollup -Sensor $currentSensors if (@($current.Files).Count -eq 0) { throw "No passing Pester file-summary lines were found. Get-Help `"$PSCommandPath`" -Parameter SensorPath" } $baseline = $null $baselinePaths = @(Expand-RankPathList -Path $BaselineSensorPath) if ($baselinePaths.Count -gt 0) { $baselineSensors = @() foreach ($inputPath in $baselinePaths) { $fullPath = Resolve-RankInputPath -InputPath $inputPath -RepoRoot $repoRoot $baselineSensors += Read-RankSensor -Path $fullPath } $baseline = Get-RankRollup -Sensor $baselineSensors } $ordered = @(Get-RankOrderedFile -File $current.Files -SortName $Sort -MinimumSeconds $MinFileSeconds) $heldOut = @($current.Files).Count - @($ordered).Count $shown = $ordered if ($Top -gt 0) { $shown = @($ordered | Select-Object -First $Top) } Write-CliSection -Title "Harness file times" -Style "info" if ($null -ne $discoveredNote) { Write-CliKeyValue -Label "discovery" -Value "newest full-suite sensors" -LabelWidth 14 } Show-RankSensorList -Title "Current" -Sensor $current.Sensors if ($null -ne $baseline) { Show-RankSensorList -Title "Baseline" -Sensor $baseline.Sensors } Write-CliColor -Message "" -Literal Show-RankTable -Row $shown Show-RankSummary -Current $current -Baseline $baseline -Shown $shown -HeldOut $heldOut -SortName $Sort -Opened $(if ($null -ne $discoveredNote) { $discoveredNote.Opened } else { 0 }) -Seen $(if ($null -ne $discoveredNote) { $discoveredNote.Seen } else { 0 }) } catch { Write-CliSection -Title "Rank harness file times failed" -Style "error" Write-CliStatus -State "Error" -Message $_.Exception.Message if ($_.InvocationInfo -and $_.InvocationInfo.PositionMessage) { Write-CliColor -Message $_.InvocationInfo.PositionMessage -Literal } exit 2 } finally { Write-CliEnd } |