src/PSMutation.Runner.ps1
|
<# .SYNOPSIS Execution engine for the PowerShell mutation runner - baseline, candidate selection, and per-mutant Pester runs. Operates entirely on SANDBOX paths (see PSMutation.Sandbox.ps1); tracked source is never touched. .DESCRIPTION Depends on PSMutation.Operators.ps1 for candidates and PSMutation.Pester.ps1 for the child runspace's import contract. Each function is small and single-purpose so every unit stays under the complexity ceiling. Each mutant's covering tests run in a cancellable runspace under a wall-clock timeout: the loop-condition guard is a speed optimisation that avoids obviously-doomed condition mutants, but the timeout is the real safety net -- a mutated loop *body* can still make a guarded loop never terminate, and Stop() interrupts it so the run never hangs. Mutants are dispatched across a pool of WORKERS, each with its own sandbox copy and its own Pester-loaded runspace. A serial run is a pool of one through the same scheduler rather than a separate path, and finished mutants are retired in candidate order, so the report does not depend on which worker happened to finish first. #> function Set-PSMutationCoverageBreakpoint { # Ask for breakpoint coverage where the Pester has the choice (#270). Feature-detected rather than # version-compared, as SkipRemainingOnFailure is: Pester 5.2.2 has no UseBreakpoints setting # (measured) because it only ever had breakpoints, and assigning a property that is not there # would fail the run. Its own function so both arms are tested against stand-ins -- every Pester # CI installs for the suite has the setting. [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Sets a field on an in-memory configuration object.')] [OutputType([void])] [CmdletBinding()] param([Parameter(Mandatory)] $CodeCoverage) if ($CodeCoverage.PSObject.Properties['UseBreakpoints']) { $CodeCoverage.UseBreakpoints = $true } } function Invoke-PSMutationBaseline { <# .SYNOPSIS Run the suite once (green-gate) and capture per-file covered line numbers, so we only mutate lines a test actually exercises (Stryker's perTest idea). .OUTPUTS @{ Passed = <bool>; DurationSeconds = <double>; CoveredLines = @{ file = HashSet[int] }; FailedTest = <string[]> } #> [OutputType([hashtable])] [CmdletBinding()] param( [Parameter(Mandatory)] [string[]]$TestPath, # AllowEmptyCollection: a -ChangedFile run over a docs-only pull request has nothing to # mutate and still needs its green gate. A mandatory [string[]] refuses an empty array # outright, which turned that ordinary case into a binding failure. [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]]$MutateFiles, # Where Pester's coverage XML goes. Mandatory rather than defaulted to temp: the sandbox is # the one directory this run owns and disposes of, and a default would put the file back in # shared temp for any caller who forgot. [Parameter(Mandatory)] [string]$SandboxRoot, # Instrument for coverage. OFF by default, because the green gate is what every run needs # and the covered lines are what only some do -- see Test-PSMutationCoverageNeeded. A # baseline without it returns an empty CoveredLines, which is the true answer: nothing was # measured. It must not then be handed to a filter, and one decision drives both. [switch]$Coverage ) $cfg = New-PesterConfiguration $cfg.Run.Path = $TestPath $cfg.Run.PassThru = $true $cfg.Output.Verbosity = 'None' $cfg.CodeCoverage.Enabled = [bool]$Coverage $cfg.CodeCoverage.Path = $MutateFiles # Coverage is read from the RESULT OBJECT below; this file is never opened. The path exists # only because Pester writes one somewhere, and its default is a coverage.xml in the working # tree -- which is the thing to avoid. # # It goes in the SANDBOX, not temp. In temp it was "psmut-coverage-$PID.xml", which nothing # ever deleted: the startup sweep matches directories named psmut-sandbox-*, so it could not # match this file by construction, and they accumulated for the life of the machine -- 67 of # them on the box this was found on. The sandbox is already removed in the run's finally, so # putting it there makes the cleanup the one that already exists rather than a second one to # keep in step. # # It also stops a predictable write into world-writable temp, which is the same shape as the # sandbox path that was fixed in #95 -- lesser, because the content is coverage data rather # than source, but there is no reason to keep one after removing the other. $cfg.CodeCoverage.OutputPath = Join-Path $SandboxRoot 'coverage.xml' # BREAKPOINTS, not Pester 6's default Profiler tracer (#270). The tracer fails any test that # touches a PowerShell class -- "Index was out of range", measured with a class in a module and # in a dot-sourced script alike -- so a baseline over class-based code was refused as "not # green" with the tests blamed. Measured on this repo's own baseline, breakpoints also saw ONE # MORE executed command than the tracer (2534 against 2533), for 6% more time (69s against # 65s): the tracer is not only fragile but short. Set-PSMutationCoverageBreakpoint -CodeCoverage $cfg.CodeCoverage $sw = [System.Diagnostics.Stopwatch]::StartNew() $result = Invoke-Pester -Configuration $cfg $sw.Stop() # A foreach STATEMENT, not a pipeline, and that is the whole of a bug this uncovered. With the # tracer off Pester reports no CommandsExecuted, and `$null | ForEach-Object` runs its body # ONCE with $_ = $null -- so GetFullPath was handed an empty string and the run died inside # the baseline. Wrapping in @( ) does not help: @($null) has one element, which is $null. # `foreach ($x in $null)` iterates zero times, which is the answer this wants. Measured all # three; the numbers are 1, 1 and 0. # # It was unreachable while coverage was unconditional, which is why it sat here until the # tracer became optional. # # Reached through Get-PSMutationCoveredLine, which reads the MISSED commands as well as the # executed ones: a line is covered by the innermost command spanning it, and only the two lists # together say which command that is and whether it ran. $covered = Get-PSMutationCoveredLine -Executed $result.CodeCoverage.CommandsExecuted ` -Missed $result.CodeCoverage.CommandsMissed # The reason each failure gave, decided ONCE. The names the guard prints and the count it reads # to decide whether to add a hint are two views of this list; computed apart they could disagree # about what "no reason" means, and the hint would fire beside a name that carried one. # A foreach STATEMENT, because Failed is null on a green run and a pipeline over it would run # once with $null and report a phantom failure. $failures = [System.Collections.Generic.List[object]]::new() foreach ($t in $result.Failed) { $failures.Add([pscustomobject]@{ Name = [string]$t.ExpandedPath; Why = (Get-PSMutationFailedTestReason -Test $t) }) } return @{ Passed = ($result.Result -eq 'Passed') # Read by the guard only when EVERY failure is unexplained: then the names alone point at # nothing, and the reader needs to know where else to look. Unexplained = @($failures | Where-Object { -not $_.Why }).Count DurationSeconds = $sw.Elapsed.TotalSeconds CoveredLines = $covered # Carried so the guard can NAME what broke, and say why. A red baseline is reported by # a gate that runs -Quiet, where the whole run prints one line -- so "not green", or # even a bare test name, sends the reader to reproduce a failure that by definition is # not happening on the machine they are standing on. # # NO dangling separator when there is nothing to say: "Some.Test -- ." reads as a reason # that was lost rather than one that never existed. What counts as a reason is decided in # Get-PSMutationFailedTestReason. FailedTest = @(foreach ($f in $failures) { $f.Why ? "$($f.Name) -- $($f.Why)" : $f.Name }) } } function Get-PSMutationFailedTestReason { <# .SYNOPSIS The one line a failed Pester test gives as its reason, or '' when it gives none. .DESCRIPTION ErrorRecord is a COLLECTION, not one record -- a test can fail for more than one reason, and Pester hands back a List. Reading `.Exception.Message` off the list itself works only by member enumeration, which yields the inner value for a ONE-element list and NOTHING for a longer one, so the reason came back '' exactly when a test had several errors. The FIRST record is the reason: later ones are usually cascade from it. Of that record, the first NON-EMPTY line. A Pester message is an expectation, then the actual, then a stack; the second line would report "at <ScriptBlock>", which is a fact about nothing, and a leading blank line would report ''. Split on \r?\n rather than on a bare newline: a CRLF message otherwise keeps a trailing carriage return, which prints as a stray line break in the middle of the gate's one line of output. A test can carry no record at all: when a BeforeAll dies under ErrorActionPreference = Stop, Pester marks every test in the block Failed and attaches the error to the CONTAINER. The container's record is deliberately NOT reached for -- it holds Pester's own break/continue guard text, which says nothing about the consumer's failure. '' is the honest answer. Walked with foreach rather than indexed: the empty case is precisely the one this has to answer, and indexing an empty collection is an error wherever StrictMode is on. #> [OutputType([string])] [CmdletBinding()] param([Parameter(Mandatory)] $Test) foreach ($record in @($Test.ErrorRecord)) { foreach ($line in ([string]$record.Exception.Message -split "\r?\n")) { if (-not [string]::IsNullOrWhiteSpace($line)) { return $line.Trim() } } return '' } return '' } function Assert-PSMutationBaselineGreen { # Refuse to mutate against a failing suite. Every mutant would "die" for the # reason the suite was already red, producing a perfect score that means nothing # -- the single most misleading result this tool could hand back. # # Lives beside Invoke-PSMutationBaseline, whose output it is the only reader of. [CmdletBinding()] param([Parameter(Mandatory)] $Baseline) if ($Baseline.Passed) { return } # Named, and capped at ten. A wholly broken suite would otherwise paste hundreds of names # into one exception and bury the first, which is the one most likely to explain the rest. $named = @($Baseline.FailedTest) $detail = '' if ($named.Count -gt 0) { $shown = @($named | Select-Object -First 10) $more = '' if ($named.Count -gt $shown.Count) { $more = " (and $($named.Count - $shown.Count) more)" } $detail = " Failed: $($shown -join '; ')$more." } # Every failure without a reason is a different situation from a failing assertion, and the # names alone send the reader to the tests. Two causes produce it, and neither is in the tests' # own lines: a BeforeAll that died, whose error Pester attaches to the FILE, and a damaged # Pester install, which fails even a test with no assertion in it. Both have been seen. if ($named.Count -gt 0 -and $Baseline.Unexplained -eq $named.Count) { $detail += ' None of them carried an error. A BeforeAll that failed attaches its error to the test file rather than to the tests, so run that file alone to see it. If even a test with no assertions fails this way, the Pester installation itself may be damaged: reinstall it with Install-Module Pester -RequiredVersion <the version in use> -Force.' } throw "Baseline suite is not green - fix the tests before mutating.$detail" } function Get-PSMutationCoveredLine { <# .SYNOPSIS Per-file covered line numbers, from the commands Pester saw run and the ones it saw not run. Pure. .DESCRIPTION A command is attributed to the line it STARTS on. Taken alone that leaves every later line of a statement written over several lines uncovered although the statement ran -- the continuation of a long condition, an argument list, a message built from several strings -- so `coveredLinesOnly` dropped every mutant on those lines from the score in silence, while the coverage gate, which counts commands, reported 100% over the same code. So a line no command starts on is covered when the INNERMOST command spanning it ran. Not "any command spanning it": a pipeline whose script block never executed still spans that block's lines, and reading those as covered would hand mutants nothing ran to the loop, to come back as survivors no test could have killed. The innermost spanning command is the one whose verdict belongs to the line, which is why the MISSED commands are read too: they are what stops a hit outer command speaking for an inner one that did not run. A line a command starts on keeps its old answer -- covered when any command starting there ran. A line no command spans at all, such as a `param()` default, stays uncovered: nothing Pester instruments can say whether it executed. .PARAMETER Executed Pester's CommandsExecuted. Null when the tracer was off, and then nothing is covered. .PARAMETER Missed Pester's CommandsMissed. #> [OutputType([hashtable])] [CmdletBinding()] param( [AllowNull()] [AllowEmptyCollection()] [object[]]$Executed, [AllowNull()] [AllowEmptyCollection()] [object[]]$Missed ) # foreach STATEMENTS throughout: a null list iterates zero times, where a pipeline would run # once with $null and hand GetFullPath an empty string. $byFile = @{} foreach ($pass in @(@{ Hit = $true; Commands = $Executed }, @{ Hit = $false; Commands = $Missed })) { foreach ($c in $pass.Commands) { $f = [System.IO.Path]::GetFullPath($c.File) if (-not $byFile.ContainsKey($f)) { $byFile[$f] = [System.Collections.Generic.List[object]]::new() } $byFile[$f].Add([pscustomobject]@{ StartLine = [int]$c.StartLine; StartColumn = [int]$c.StartColumn EndLine = [int]$c.EndLine; Hit = $pass.Hit }) } } $covered = @{} foreach ($f in $byFile.Keys) { $covered[$f] = Get-PSMutationFileCoveredLine -Command $byFile[$f].ToArray() } return $covered } function Get-PSMutationFileCoveredLine { <# .SYNOPSIS The covered lines of ONE file, from its commands. Pure. .DESCRIPTION One pass over the lines with a stack of the commands still open. Extents from one parse are nested or disjoint, so after the commands that ended above a line are popped, the top of the stack is the innermost command spanning it -- no per-line search over every command, which on a large file is the difference between linear and quadratic. A command that ended earlier on the same line as a later sibling can sit beneath it on the stack. It never decides a line: it ended on the line it was pushed on, so it is popped before any later line is judged. #> # Both types, as for every comma-wrapped return here: `, $set` is statically an Object[] # wrapper that PowerShell unrolls on the way out, and the caller receives the set. [OutputType([System.Collections.Generic.HashSet[int]], [object[]])] [CmdletBinding()] param([Parameter(Mandatory)] [object[]]$Command) $covered = [System.Collections.Generic.HashSet[int]]::new() # Sorted, because Pester hands executed and missed back as two lists and neither order is a # promise. The stack only works over commands in document order. $sorted = @($Command | Sort-Object StartLine, StartColumn) $last = ($sorted | Measure-Object -Property EndLine -Maximum).Maximum $open = [System.Collections.Generic.Stack[object]]::new() $next = 0 for ($line = $sorted[0].StartLine; $line -le $last; $line++) { while ($open.Count -gt 0 -and $open.Peek().EndLine -lt $line) { [void]$open.Pop() } # Every command starting here: a start line is covered when ANY of them ran, which is the # rule this function inherits unchanged. while ($next -lt $sorted.Count -and $sorted[$next].StartLine -eq $line) { if ($sorted[$next].Hit) { [void]$covered.Add($line) } $open.Push($sorted[$next]) $next++ } # Every other line: the innermost command spanning it decides. On a start line this is # the command just pushed, which the loop above has already answered for. if ($open.Count -gt 0 -and $open.Peek().Hit) { [void]$covered.Add($line) } } return , $covered } function Test-PSMutantCovered { # True if a candidate's line was executed by the baseline run. Pure. [OutputType([bool])] [CmdletBinding()] param([Parameter(Mandatory)] $Candidate, [Parameter(Mandatory)] [hashtable]$CoveredLines) $f = [System.IO.Path]::GetFullPath($Candidate.File) return $CoveredLines.ContainsKey($f) -and $CoveredLines[$f].Contains([int]$Candidate.Line) } function Get-PSMutationRunBaseline { <# .SYNOPSIS The baseline this run will size its budget from -- measured, or the stand-in a preview uses. .DESCRIPTION A run must measure: it needs a green suite to mutate against and a duration to derive the per-mutant timeout from. A preview with no coverage filter to satisfy needs neither, and a suite run would buy it nothing. The stand-in is a FUNCTION rather than two lines inside an else, because its zeros are otherwise unobservable: nothing in a preview reads a duration, so every mutant of them survives a test that can only assert the preview did not throw. Returned from a named function, the contract -- no time taken, no coverage measured -- is something a test can state. It carries no Passed flag. A green verdict is the one thing this did not establish, and fabricating one here is exactly the "a number nobody measured, reported as one" failure the module exists to catch. #> [OutputType([pscustomobject])] [CmdletBinding()] param( [Parameter(Mandatory)] $Plan, [Parameter(Mandatory)] [string]$SandboxRoot, [switch]$Measure, [switch]$Coverage, [switch]$Quiet ) if (-not $Measure) { return [pscustomobject]@{ DurationSeconds = 0.0; CoveredLines = @{} } } Write-PSMutationOutput -Quiet:$Quiet -Lines (New-PSMutationLine -Role 'Banner' ` -Text "`nPSMutant - PowerShell mutation testing (sandboxed)`n Running baseline suite...") $baseline = Invoke-PSMutationBaseline -TestPath $Plan.AllTests -MutateFiles $Plan.Mutate ` -SandboxRoot $SandboxRoot -Coverage:$Coverage Assert-PSMutationBaselineGreen -Baseline $baseline return $baseline } function ConvertTo-PSMutationDisplayPerFile { <# .SYNOPSIS The per-file tally with its paths made repo-relative, for anything a human reads. .DESCRIPTION Select-PSMutationCandidate works in the SANDBOX, so every File it reports is an absolute path under a temp directory whose name changes on every run. That is the right value to mutate and the wrong one to print or to persist: it cannot be matched against a checkout, it differs between two runs of the same commit, and it names a directory that is deleted before the reader sees it. It reached the report's `filesWithNoMutants` and the summary's uncovered caveat that way. Converted ONCE, here, rather than at each of the places that display it -- the candidates themselves keep their sandbox paths, because that is where the loop reads them from. #> [OutputType([object[]])] [CmdletBinding()] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]]$PerFile, [Parameter(Mandatory)] [string]$SandboxRoot ) return , @($PerFile | ForEach-Object { [pscustomobject]@{ File = (ConvertFrom-PSMutationSandboxPath -Path $_.File -SandboxRoot $SandboxRoot) Produced = $_.Produced Kept = $_.Kept ByOperator = $_.ByOperator } }) } function Get-PSMutationCandidateByOperator { # One file's candidate counts, split by the operator that produced them. # # BOTH numbers per operator, because they answer different questions and only the pair # locates the fault. `Produced` says which operators matched the file at all -- zero across # the board means the file has nothing this module knows how to mutate, and no amount of # test-writing changes that. `Kept` says how many survived the coverage filter -- produced # but not kept means the mapped suite does not reach those lines, which IS a test problem. # Reported as one number, a file scoring a vacuous 100% looks the same either way. [OutputType([System.Collections.Specialized.OrderedDictionary])] [CmdletBinding()] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]]$Produced, [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]]$Kept ) $byOp = [ordered]@{} foreach ($c in $Produced) { if (-not $byOp.Contains($c.Operator)) { $byOp[$c.Operator] = @{ Produced = 0; Kept = 0 } } $byOp[$c.Operator].Produced++ } # No guard on the key: Kept is a subset of Produced by construction, so a missing key here # would mean the caller passed two unrelated sets, which is a fault to surface rather than # to absorb into a zero. foreach ($c in $Kept) { $byOp[$c.Operator].Kept++ } return $byOp } function Select-PSMutationCandidate { # Enumerate candidates across the mutate files, keeping only covered ones (opt), and # report what that removed. # # Returns BOTH, rather than only the survivors, because the coverage filter can empty a # whole mutate file and the score then answers for a smaller set than the config asked # for -- upward, and silently. It fires the moment a file joins `mutate` before its tests # exist, or a refactor stops a suite exercising a module. The per-file tally is the only # place the pre-filter count still exists; recomputing it later would mean parsing every # file a second time. [OutputType([pscustomobject])] [CmdletBinding()] param( # AllowEmptyCollection, like the baseline's: a -ChangedFile run over a docs-only pull # request has no file to enumerate, and that is an ordinary outcome rather than a # binding failure. It yields no candidates, which is the true answer. [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]]$MutateFiles, [Parameter(Mandatory)] [string[]]$Operators, [bool]$CoveredLinesOnly, $CoveredLines ) $out = [System.Collections.Generic.List[object]]::new() $perFile = [System.Collections.Generic.List[object]]::new() foreach ($file in $MutateFiles) { $produced = @(Get-PSMutationCandidate -Path $file -Operators $Operators) $kept = @($produced | Where-Object { -not $CoveredLinesOnly -or (Test-PSMutantCovered -Candidate $_ -CoveredLines $CoveredLines) }) foreach ($c in $kept) { $out.Add($c) } $perFile.Add([pscustomobject]@{ File = $file; Produced = $produced.Count; Kept = $kept.Count ByOperator = (Get-PSMutationCandidateByOperator -Produced $produced -Kept $kept) }) } return [pscustomobject]@{ Candidates = $out.ToArray(); PerFile = $perFile.ToArray() } } # The outcomes this module understands from a covering-test run. Pester's run-level result # supplies 'Passed' and 'Failed'; 'TimedOut' is minted here by Stop-PSMutationJob. Anything # outside this set is refused rather than scored -- see Get-PSMutationVerdict. $script:PSMutationKnownOutcomes = @('Passed', 'Failed', 'TimedOut') function Get-PSMutationSuiteShell { # One worker's warm shell, loaded with the child script and its arguments but NOT started. # # Shared by a mutant and by the green check before the loop, so the check runs the suite # exactly the way a mutant will: same runspace, same script, same parameters. Not started, # because a mutant must write its file between preparing the shell and invoking it -- see # Start-PSMutantEvaluation for why that order is load-bearing. [OutputType([powershell])] [CmdletBinding()] param( [Parameter(Mandatory)] [int]$WorkerId, [Parameter(Mandatory)] [string[]]$Tests, # The worker's own sandbox: modules loaded from it are removed before each run (#249). [Parameter(Mandatory)] [string]$SandboxRoot, [switch]$RecordAllKillers ) $ps = Get-PSMutationWarmShell -WorkerId $WorkerId $ps.Commands.Clear() [void]$ps.AddScript((Get-PSMutationWarmPesterScript).ToString()).AddParameter('tests', $Tests). AddParameter('recordAllKillers', [bool]$RecordAllKillers).AddParameter('sandboxRoot', $SandboxRoot) return $ps } function Start-PSMutantEvaluation { <# .SYNOPSIS Splice one mutant into ITS WORKER'S sandbox file and set the covering tests running, without waiting for them. .DESCRIPTION Half of what used to be one synchronous function, and the split is what makes parallel evaluation possible: the caller can have one of these in flight per worker and collect whichever finishes first. There is exactly ONE execution path. A serial run is a pool of one worker through this same pair, rather than a second, simpler route -- two paths would be two places for the verdict to be decided, and they would disagree in whichever case nobody tests. The ORDER of the last two statements is load-bearing. The shell is obtained BEFORE the file is written, so a Pester import that fails leaves the sandbox exactly as it was found. The old shape wrote first and leaned on a `finally` to undo it; this needs no undo at all. .OUTPUTS The in-flight job: the worker it belongs to, its handle, and everything needed to collect it and put the file back. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Writes only inside the throwaway sandbox; tracked source is never touched.')] [OutputType([pscustomobject])] [CmdletBinding()] param( # The mutant addressed in the WORKER'S sandbox -- this is the file that gets written. [Parameter(Mandatory)] $Candidate, # The same mutant addressed in the PRIMARY sandbox, which is what the report says. Carried # because a row must not name a per-worker temp directory: which worker ran a mutant is # scheduling, and scheduling must not be visible in the answer. [Parameter(Mandatory)] $Source, [Parameter(Mandatory)] [int]$Index, [Parameter(Mandatory)] [string]$MutatedContent, [Parameter(Mandatory)] [AllowEmptyString()] [string]$OriginalContent, [Parameter(Mandatory)] [string[]]$CoveringTests, [Parameter(Mandatory)] [int]$WorkerId, # The sandbox this worker mutates, whose modules the child unloads before running. [Parameter(Mandatory)] [string]$SandboxRoot, [switch]$RecordAllKillers ) $ps = Get-PSMutationSuiteShell -WorkerId $WorkerId -Tests $CoveringTests -SandboxRoot $SandboxRoot -RecordAllKillers:$RecordAllKillers [System.IO.File]::WriteAllText($Candidate.File, $MutatedContent) # The clock starts at BeginInvoke rather than when the candidate was queued, so a mutant's # measured seconds are what it spent running and not what it spent waiting for a worker. # The per-mutant budget is a statement about the child, and a queue time folded into it would # turn a busy pool into a run full of timeouts. return [pscustomobject]@{ WorkerId = $WorkerId; Shell = $ps; Async = $ps.BeginInvoke() Candidate = $Candidate; Source = $Source; Index = $Index OriginalContent = $OriginalContent Clock = [System.Diagnostics.Stopwatch]::StartNew() } } function Stop-PSMutationJob { # Cut off a mutant that outlived its budget, and discard the runspace it was using. # # Stop() leaves a runspace unusable, so it is discarded rather than handed to the next mutant. # A timeout is rare; paying a cold start after one is the cheap half of the trade -- and it is # now one WORKER'S cold start rather than the pool's. [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Stops an in-process pipeline; there is no system state to confirm.')] [OutputType([pscustomobject])] [CmdletBinding()] param([Parameter(Mandatory)] $Job) # Requested, not awaited (#250): a synchronous Stop() waits for a blocking .NET call to return, # and while it waited every other worker stalled with it. Stop-PSMutationWarmRunspace -WorkerId $Job.WorkerId # No killers on a timeout, and an EMPTY list rather than none: the caller reads .Killers # unconditionally, and a missing property would make "nothing killed it" and "we never # looked" the same answer at the one point where they differ most. return [pscustomobject]@{ Result = 'TimedOut'; Killers = @() } } function Receive-PSMutationJob { # The finished child's verdict. # # A child that returned no verdict proved nothing about the mutant. Handing that back would # classify it Killed -- anything but 'Passed' is a kill -- so a broken child reads as a # perfect score. That is exactly how the Pester version collision stayed invisible for so # long. Fail the run instead of scoring it. [OutputType([pscustomobject])] [CmdletBinding()] param([Parameter(Mandatory)] $Job) $result = $Job.Shell.EndInvoke($Job.Async) | Select-Object -Last 1 $outcome = [string]$result.Result if (-not $outcome) { $why = Get-PSMutationRunspaceError -Runspace $Job.Shell Close-PSMutationWarmRunspace -WorkerId $Job.WorkerId throw "The covering tests produced no result: $why" } return [pscustomobject]@{ Result = $outcome; Killers = @($result.Killers) } } function Get-PSMutationVerdict { <# .SYNOPSIS What one covering-test outcome says about the mutant. Pure. .OUTPUTS 'Killed' | 'Survived' | 'TimedOut' -- Survived only if the suite still fully passes. A timeout scores WITH the kills, because a mutant that hangs the suite is a fault, but it is reported apart from them: "the suite proved this fault is caught" and "the suite hung and we assumed so" are different claims and only the first is evidence. Folded together, a suite that is merely too slow inflates the score. #> [OutputType([pscustomobject])] [CmdletBinding()] param([Parameter(Mandatory)] $Run) $outcome = $Run.Result # The verdict and the killers travel together from here, so a caller cannot record one # without the other. Survived and TimedOut carry an empty list rather than none, because # "nothing killed it" and "we did not look" must not be the same value. if ($outcome -eq 'Passed') { return [pscustomobject]@{ Status = 'Survived'; Killers = @() } } if ($outcome -eq 'TimedOut') { return [pscustomobject]@{ Status = 'TimedOut'; Killers = @() } } # A CLOSED vocabulary. Everything above is a value this module understands; anything # else is an outcome nobody modelled, and the fall-through below scores it Killed -- # toward the flattering answer, silently, with no test failing. # # The collapse is correct for every shipping Pester, whose run-level result is # two-valued. The risk is a WIDENED vocabulary rather than a renamed one: a rename # fails loudly at the baseline, which compares against the literal 'Passed', but a # third state that coexists with it leaves the baseline green and scores every mutant # returning it as killed. That is a perfect score over tests that proved nothing -- # the same shape as the Pester-collision bug, reached through a door its fix left open. if ($outcome -notin $script:PSMutationKnownOutcomes) { throw ("The covering tests returned an outcome this version of PSMutant does not " + "model: '$outcome'. Known outcomes are $($script:PSMutationKnownOutcomes -join ', '). " + "Scoring it would guess, and the guess flatters the score.") } return [pscustomobject]@{ Status = 'Killed'; Killers = @($Run.Killers) } } function Complete-PSMutantEvaluation { <# .SYNOPSIS Collect one dispatched mutant's verdict and put its sandbox file back. .DESCRIPTION -Expired is the SCHEDULER'S decision, not this function's, because only the scheduler knows what else is in flight: it waits on every handle at once and hands back whichever it found finished or past its budget. Asking here would mean a second clock reading and a second answer to a question already settled. .OUTPUTS @{ Status; Killers } -- see Get-PSMutationVerdict. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Writes only inside the throwaway sandbox; tracked source is never touched.')] [OutputType([pscustomobject])] [CmdletBinding()] param([Parameter(Mandatory)] $Job, [switch]$Expired) try { # A job decided without a run carries its verdict; there is nothing to receive or stop. if ($Job.PSObject.Properties['Verdict']) { return $Job.Verdict } return Get-PSMutationVerdict -Run ($Expired ? (Stop-PSMutationJob -Job $Job) : (Receive-PSMutationJob -Job $Job)) } finally { # Still restored per mutant, and deliberately. The next mutant on this worker writes the # whole file anyway, so this write is redundant between two mutants of the SAME file in # the SAME worker -- but it is what makes the pair self-contained: a mutant that throws, # or a run killed here, leaves the sandbox as it found it. [System.IO.File]::WriteAllText($Job.Candidate.File, $Job.OriginalContent) # The covering tests too, when they were marked for this mutant (#270). A foreach STATEMENT, # so a job with none -- the ordinary case -- iterates zero times rather than once on $null. foreach ($o in $Job.TestOriginals) { [System.IO.File]::WriteAllBytes($o.Path, $o.Bytes) } } } function Get-PSMutationProgressLine { # One per-mutant progress line. Pure, and emitted as the loop goes rather than # collected: a run of several hundred mutants takes minutes, and a progress report # delivered at the end is not a progress report. [OutputType([pscustomobject])] [CmdletBinding()] param([int]$Index, [int]$Total, $Result, [string]$DisplayFile) $survived = $Result.Status -eq 'Survived' # '!' for a mutant with no verdict: neither the '.' of a survivor nor the 'x' of a kill. $unscored = $Result.Status -in 'CompileError', 'RuntimeError' $glyph = $unscored ? '!' : ($survived ? '.' : 'x') $role = ($survived -or $unscored) ? 'Warn' : 'Muted' return New-PSMutationLine -Role $role -Data $Result ` -Text (" [{0}/{1}] {2} {3}:{4} {5}" -f $Index, $Total, $glyph, $DisplayFile, $Result.Line, $Result.Description) } function Get-PSMutationStalledFault { <# .SYNOPSIS The fault, if any, when one mutant's wall clock says the run stopped running. #> [OutputType([string])] [CmdletBinding()] param( [Parameter(Mandatory)] [double]$MutantSeconds, [Parameter(Mandatory)] [int]$TimeoutSeconds, [Parameter(Mandatory)] [int]$Index, [Parameter(Mandatory)] [int]$Total ) # The precise version of the run bound, and the one that fires within a single mutant instead # of at the end of a budget nobody wants to wait out. Every mutant is already bounded: the # child is given TimeoutSeconds and its handle is waited on for exactly that. So a mutant # whose WALL CLOCK is far past its own budget did not run slowly -- the mechanism that was # supposed to stop it did not fire. # # That is what an overnight hang looks like from inside: 875 minutes elapsed against 333 # seconds of CPU, because the machine slept mid-run and the handle never came back. Comparing # the two numbers names the cause; a total-run deadline only reports the symptom, hours later. # # x4 and a +30s floor, so an ordinary overrun cannot trip it. A timed-out mutant already # costs its full budget plus the cost of discarding and rebuilding the runspace, and a loaded # machine can stretch that; four times the budget is not something a working run reaches. $limit = [math]::Max(($TimeoutSeconds * 4), ($TimeoutSeconds + 30)) if ($MutantSeconds -le $limit) { return $null } return ("Mutant $Index of $Total took $([int]$MutantSeconds)s against a per-mutant budget of " + "${TimeoutSeconds}s. The bound on the child did not fire, which is what a suspended or " + 'wedged run looks like rather than a slow one -- a machine that slept mid-run leaves the ' + 'handle it was waiting on never signalling. Stopping here so the partial report says how ' + 'far the run got.') } function Get-PSMutationOverBudgetFault { <# .SYNOPSIS The fault, if any, when the whole run has outlived its wall-clock budget. #> [OutputType([string])] [CmdletBinding()] param( [Parameter(Mandatory)] [double]$ElapsedSeconds, [Parameter(Mandatory)] [int]$DeadlineSeconds, [Parameter(Mandatory)] [int]$Index, [Parameter(Mandatory)] [int]$Total ) # A pure decision rather than an inline comparison on a stopwatch, so its BOUNDARY can be # tested. Written inline it could not: elapsed wall-clock never lands exactly on the budget, # so nothing could tell -gt from -ge and self-mutation said so. if ($DeadlineSeconds -le 0) { return $null } if ($ElapsedSeconds -le $DeadlineSeconds) { return $null } return ("This run passed its wall-clock budget of ${DeadlineSeconds}s after $Index of " + "$Total mutant(s). Every mutant is bounded and the run was not, so a suspended or wedged " + 'run used to sit there indefinitely and look exactly like a slow one. Raise ' + 'runTimeoutSeconds, or set it to 0 if something else already kills wedged runs.') } function Get-PSMutationCoveringSuite { <# .SYNOPSIS The test files that cover one mutate file: its own mapping, or the whole suite. #> # BOTH types, and the second is the price of the comma-wrap: `, $x` is statically an # Object[] wrapper that PowerShell unrolls on return, so PSUseOutputTypeCorrectly # contradicts a bare [string[]]. Declaring only [object[]] would satisfy the analyzer # and stop documenting what a caller actually receives. [OutputType([string[]], [object[]])] [CmdletBinding()] param( [Parameter(Mandatory)] [string]$File, [Parameter(Mandatory)] [hashtable]$TestsByFile, [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]]$AllTests ) # ONE place, because the loop and the verbose trace both need the answer and a second copy of # a fallback is a second thing to get wrong -- the trace's copy survived its own mutant, since # nothing asserted what it printed. # Comma-wrapped for the reason the report's two sibling collectors are: an empty result # would unroll to $null, and this feeds a -TestPath that would then bind nothing. $suite = [string[]]@($TestsByFile.ContainsKey($File) ? $TestsByFile[$File] : $AllTests) return , $suite } function New-PSMutationResultRow { <# .SYNOPSIS One mutant's result as the report publishes it. Pure. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Pure projection: returns an object, changes no state.')] [OutputType([pscustomobject])] [CmdletBinding()] param( [Parameter(Mandatory)] $Candidate, [Parameter(Mandatory)] $Verdict, [Parameter(Mandatory)] [string]$DisplayFile ) return [pscustomobject]@{ # Function is carried so an equivalence declaration can address this mutant by # the function it lives in rather than by a line number, which moves whenever # anything above the mutant is edited and takes the declaration stale with it. # # This row is the report's published shape. Adding a field widens what every # consumer may depend on, so a test asserts the exact list: widening should # cost a deliberate edit, not happen as a side effect of an internal rename. # # Which WORKER ran the mutant is deliberately absent. It is scheduling, it differs # between two runs that must be identical, and a field that changes with the machine # is one a consumer would eventually diff. Id = $Candidate.Id; Function = $Candidate.Function; File = $DisplayFile; Line = $Candidate.Line Operator = $Candidate.Operator; Description = $Candidate.Description; Status = $Verdict.Status # The tests that noticed. TRUNCATED under the default configuration and complete under # -RecordAllKillers, which is stated at run level rather than inferred from the length # of this list. # # Truncated is not the same as "exactly one", measured: over 118 killed mutants the # default still reported more than one killer for 20 of them, because several tests can # be marked failed before Pester's early stop takes hold. So the length here says # nothing about how many tests really kill a mutant -- the same run with every killer # recorded found 85. Read killersComplete, never the count. KilledBy = @($Verdict.Killers) } } function Get-PSMutationFreeWorker { # The lowest-numbered worker with nothing in flight, or -1 when every worker is busy. Pure. # # LOWEST rather than any, so a serial run and a parallel one dispatch the same way for as # long as the pool is idle, and so the choice is a fact about the schedule rather than about # whichever enumeration order a hashtable felt like today. [OutputType([int])] [CmdletBinding()] # [AllowNull()] beside the mandatory, and it is not decoration. An IDLE POOL is an array whose # every element is $null, and PowerShell's mandatory check unwraps a single-element collection # before testing it: a one-worker pool with nothing in flight is `@($null)`, which binds as # null and is refused. So the serial case -- the only one every existing test exercises -- # failed at the first dispatch while every parallel one bound fine. param([Parameter(Mandatory)] [AllowNull()] [AllowEmptyCollection()] [object[]]$InFlight) for ($i = 0; $i -lt $InFlight.Count; $i++) { if ($null -eq $InFlight[$i]) { return $i } } return -1 } function Get-PSMutationJobState { <# .SYNOPSIS 'Complete', 'Expired' or 'Running' for one in-flight mutant. Pure. .DESCRIPTION Completion is asked FIRST. A child that finished a hair before its budget ran out has a verdict, and reading the clock first would throw that verdict away and score the mutant Killed on a timeout it did not have -- toward the flattering answer, which is the direction this module refuses to guess in. #> [OutputType([string])] [CmdletBinding()] param( [Parameter(Mandatory)] [bool]$Completed, [Parameter(Mandatory)] [double]$ElapsedSeconds, [Parameter(Mandatory)] [int]$TimeoutSeconds ) if ($Completed) { return 'Complete' } if ($ElapsedSeconds -ge $TimeoutSeconds) { return 'Expired' } return 'Running' } function Get-PSMutationWaitBudget { <# .SYNOPSIS How long the scheduler may block before it MUST look again, in MILLISECONDS: the least budget any in-flight mutant has left. Pure. .DESCRIPTION The least, not the average and not a fixed poll interval, because a mutant is cut off on its OWN budget and one that expires while the scheduler is asleep on somebody else's clock has overrun by however long the nap was. A fixed interval would answer the same question by waking up constantly and would still be wrong at the boundary. Floored at 1ms rather than 0: a zero timeout makes WaitAny a non-blocking poll, so a mutant already past its budget would spin the scheduler at full speed for as long as it takes the sweep to reach it. #> [OutputType([int])] [CmdletBinding()] param( # What each in-flight mutant has LEFT of its own deadline (#253). Deadlines differ per # covering suite, so the soonest expiry is no longer simply the oldest job's. [Parameter(Mandatory)] [AllowEmptyCollection()] [double[]]$RemainingSeconds, # The run-wide deadline, which caps the wait: with nothing in flight it is the whole # answer, and a longer per-file deadline only means the scheduler looks again sooner. [Parameter(Mandatory)] [int]$TimeoutSeconds ) # A minimum asked of Measure-Object rather than tracked in a loop -- and that is not tidiness. # `if ($left -lt $least)` and `-le` produce identical output for every input, because assigning # an equal value changes nothing: an equivalent mutant that would have to be argued for in the # config rather than killed by a test. Asking for a minimum has no boundary to get wrong. $least = (@($RemainingSeconds) + @([double]$TimeoutSeconds) | Measure-Object -Minimum).Minimum return [int][math]::Max(1, [math]::Ceiling($least * 1000)) } function Wait-PSMutationWorker { <# .SYNOPSIS Block until an in-flight mutant finishes or the earliest per-mutant budget runs out. .DESCRIPTION There is ALWAYS something in flight when this is called, and that is an invariant of the loop rather than a hope. Every candidate is in exactly one of four states -- not yet dispatched, in flight, parked, retired -- and the loop only reaches here with at least one not retired. The next one to retire is not parked, because retirement drains every parked mutant whose turn has come; so it is either in flight, or not yet dispatched -- and in the second case dispatch stopped because every worker was busy. Both leave something in flight. It is not guarded, deliberately. A guard for a state the loop cannot reach is a branch no test can distinguish from its own absence, and it would quietly turn a future scheduling bug into a spin instead of an exception. #> [OutputType([void])] [CmdletBinding()] param([Parameter(Mandatory)] $Schedule, [Parameter(Mandatory)] $Context) $jobs = @($Schedule.InFlight | Where-Object { $null -ne $_ }) $budget = Get-PSMutationWaitBudget -TimeoutSeconds $Context.TimeoutSeconds ` -RemainingSeconds @($jobs | ForEach-Object { $_.TimeoutSeconds - $_.Clock.Elapsed.TotalSeconds }) [void][System.Threading.WaitHandle]::WaitAny( [System.Threading.WaitHandle[]]@($jobs | ForEach-Object { $_.Async.AsyncWaitHandle }), $budget) } function New-PSMutationDecidedJob { # A job whose verdict was reached without running anything: already complete, holding the # verdict, and owning no runspace. The scheduler collects it like any finished job. [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Pure: returns an object, changes no state.')] [OutputType([pscustomobject])] [CmdletBinding()] param( [Parameter(Mandatory)] $Candidate, [Parameter(Mandatory)] $Source, [Parameter(Mandatory)] [int]$Index, [Parameter(Mandatory)] [int]$WorkerId, [Parameter(Mandatory)] [AllowEmptyString()] [string]$OriginalContent, [Parameter(Mandatory)] [string]$Status ) return [pscustomobject]@{ WorkerId = $WorkerId; Shell = $null Async = [pscustomobject]@{ IsCompleted = $true; AsyncWaitHandle = [System.Threading.ManualResetEvent]::new($true) } Candidate = $Candidate; Source = $Source; Index = $Index OriginalContent = $OriginalContent # RUNNING, though nothing runs: the sweep expires a job by its clock, so a decided job whose # completion was ever lost would otherwise wait out no deadline at all. Measured in the self # gate: a mutant forcing it incomplete hung its covering run to the 306s outer deadline. Clock = [System.Diagnostics.Stopwatch]::StartNew() Verdict = [pscustomobject]@{ Status = $Status; Killers = @() } } } function Test-PSMutationDefinesType { # True when any of these files defines a PowerShell class or enum (#270). Those are the runs that # need their covering tests marked per mutant; every other run pays nothing. [OutputType([bool])] [CmdletBinding()] param([Parameter(Mandatory)] [AllowEmptyCollection()] [string[]]$Path) foreach ($p in $Path) { $ast = [System.Management.Automation.Language.Parser]::ParseFile($p, [ref]$null, [ref]$null) if ($ast.Find({ param($n) $n -is [System.Management.Automation.Language.TypeDefinitionAst] }, $true)) { return $true } } return $false } function Set-PSMutationTestMarker { <# .SYNOPSIS Append a line naming this mutant to each covering test file, and hand back the bytes to restore (#270). .DESCRIPTION PowerShell caches a compiled script by its TEXT, process-wide, and a type literal such as [MyClass] in a cached script stays bound to the first class it saw. A consumer's test file never changes between mutants and the baseline compiles it first, so every mutant inside a class was judged against the original and survived. Measured: three class mutants survived unmarked and all three were killed marked. Functions are unaffected -- they are looked up by name when called. Appended at the END, on its own line, so no line number in a failure message moves. The original BYTES are kept rather than the text, so a byte-order mark or a missing final newline is restored exactly: the report records test-file lengths, and -ResumeFrom compares them. .OUTPUTS One { Path; Bytes } per file, to restore. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Writes only inside the throwaway sandbox; tracked tests are never touched.')] [OutputType([object[]])] [CmdletBinding()] param( [Parameter(Mandatory)] [string[]]$Tests, [Parameter(Mandatory)] [string]$Id ) $marker = [System.Text.Encoding]::ASCII.GetBytes("`n# psmutant mutant $Id -- appended so PowerShell recompiles this file`n") $kept = foreach ($t in $Tests) { $bytes = [System.IO.File]::ReadAllBytes($t) [System.IO.File]::WriteAllBytes($t, [byte[]]($bytes + $marker)) [pscustomobject]@{ Path = $t; Bytes = $bytes } } return , @($kept) } function Start-PSMutationWorkerJob { # Dispatch the next candidate to one idle worker, addressed inside THAT worker's sandbox copy. [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Writes only inside the throwaway sandbox; tracked source is never touched.')] [OutputType([pscustomobject])] [CmdletBinding()] param( [Parameter(Mandatory)] $Schedule, [Parameter(Mandatory)] $Context, [Parameter(Mandatory)] [int]$WorkerId ) $c = $Context.Candidates[$Schedule.Next] # One read per FILE, not per mutant. A file contributes many candidates -- 125 for the largest # in this repo's sibling -- and every one of them re-read the same unchanged bytes to splice # against. Keyed on the PRIMARY path, so every worker shares one read: the copies are byte # identical by construction, and reading each worker's own copy would be the same bytes off # disk once per worker to produce strings that are equal. if (-not $Context.Originals.ContainsKey($c.File)) { $Context.Originals[$c.File] = [System.IO.File]::ReadAllText($c.File) } $content = $Context.Originals[$c.File] $root = $Context.Roots[$WorkerId] # A COPY, because the candidate list is shared across workers and re-rooting in place would # point every later worker at whichever sandbox happened to run it last. $exec = $c.PSObject.Copy() $exec.File = Get-PSMutationWorkerPath -Path $c.File -SandboxRoot $Context.SandboxRoot -WorkerRoot $root $covering = Get-PSMutationCoveringSuite -File $c.File -TestsByFile $Context.TestsByFile -AllTests $Context.AllTests # The TESTS are re-rooted too, and that is the half that is easy to forget. A worker running # the primary sandbox's test files would dot-source the primary sandbox's source, so every # mutant would run against unmutated code and survive -- a score of zero, arrived at silently. $tests = [string[]]@($covering | ForEach-Object { Get-PSMutationWorkerPath -Path $_ -SandboxRoot $Context.SandboxRoot -WorkerRoot $root }) # This mutant's OWN deadline (#253), sized from its covering suite; the run-wide one only # where none was measured. $timeout = $Context.TimeoutByFile.ContainsKey($c.File) ? $Context.TimeoutByFile[$c.File] : $Context.TimeoutSeconds $mutated = Set-PSMutationText -Content $content -Candidate $c # Parsed before it is written (#252). A mutant that does not parse is decided here, without a # test run, and goes through the same retirement as every other -- so the report keeps its # candidate order and the worker is free again on the next sweep. # Nothing to restore unless the run marks its tests and this mutant reaches a run. $marked = @() $job = if (-not (Test-PSMutationParsable -Content $mutated)) { New-PSMutationDecidedJob -Candidate $exec -Source $c -Index $Schedule.Next -WorkerId $WorkerId ` -OriginalContent $content -Status 'CompileError' } else { # Marked before the run and restored when it is collected (#270), only in a run whose # mutate files define a class or enum. $marked = $Context.MarkTests ? (Set-PSMutationTestMarker -Tests $tests -Id "$($c.Id)") : @() Start-PSMutantEvaluation -Candidate $exec -Source $c -Index $Schedule.Next ` -MutatedContent $mutated -OriginalContent $content ` -CoveringTests $tests -WorkerId $WorkerId -SandboxRoot $root -RecordAllKillers:$Context.RecordAllKillers } $job | Add-Member -NotePropertyName TimeoutSeconds -NotePropertyValue $timeout -Force $job | Add-Member -NotePropertyName TestOriginals -NotePropertyValue @($marked) -Force return $job } function Start-PSMutationDispatch { # Hand the next candidate to every idle worker, until one of the two runs out. [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Writes only inside the throwaway sandbox; tracked source is never touched.')] [OutputType([void])] [CmdletBinding()] param([Parameter(Mandatory)] $Schedule, [Parameter(Mandatory)] $Context) while ($Schedule.Next -lt $Context.Total) { $w = Get-PSMutationFreeWorker -InFlight $Schedule.InFlight if ($w -lt 0) { break } $Schedule.InFlight[$w] = Start-PSMutationWorkerJob -Schedule $Schedule -Context $Context -WorkerId $w $Schedule.Next++ } } function Get-PSMutationFinishedMutant { <# .SYNOPSIS One finished mutant, collected: its published row and the seconds it took. .DESCRIPTION The seconds are carried BESIDE the row rather than in it. The row's field list is the report's contract, and a wall clock is a fact about this run on this machine rather than about the mutant -- putting it in the row would make two identical runs produce different reports. #> [OutputType([pscustomobject])] [CmdletBinding()] param( [Parameter(Mandatory)] $Job, [Parameter(Mandatory)] [string]$State, [Parameter(Mandatory)] [AllowEmptyString()] [string]$SandboxRoot ) $verdict = Complete-PSMutantEvaluation -Job $Job -Expired:($State -eq 'Expired') return [pscustomobject]@{ Row = New-PSMutationResultRow -Candidate $Job.Source -Verdict $verdict ` -DisplayFile (ConvertFrom-PSMutationSandboxPath -Path $Job.Source.File -SandboxRoot $SandboxRoot) Seconds = $Job.Clock.Elapsed.TotalSeconds # The deadline it ran under, for the stall check, which asks whether THIS mutant's own # bound failed to fire. TimeoutSeconds = $Job.TimeoutSeconds } } function Complete-PSMutationSweep { # Collect every worker whose mutant has finished or run out of budget, and free its slot. [OutputType([void])] [CmdletBinding()] param([Parameter(Mandatory)] $Schedule, [Parameter(Mandatory)] $Context) # Over the JOBS, not over slot indices, and the job says which slot it came from. Written as a # `for` over `0..Count-1` the bound is a boundary nothing can observe: reading one slot past # the end yields $null, which the idle-slot guard skips, so `-lt` and `-le` produce identical # output -- an equivalent mutant that would have to be argued for rather than killed. foreach ($job in @($Schedule.InFlight | Where-Object { $null -ne $_ })) { $state = Get-PSMutationJobState -Completed $job.Async.IsCompleted ` -ElapsedSeconds $job.Clock.Elapsed.TotalSeconds -TimeoutSeconds $job.TimeoutSeconds if ($state -eq 'Running') { continue } # Freed BEFORE collecting, so a collection that throws cannot leave a slot pointing at a # job nobody will ever wait on again. $Schedule.InFlight[$job.WorkerId] = $null $Schedule.Parked[$job.Index] = Get-PSMutationFinishedMutant -Job $job -State $state -SandboxRoot $Context.SandboxRoot } } function Get-PSMutationLoopFault { <# .SYNOPSIS The fault, if any, that should stop the run after one mutant is retired. .DESCRIPTION Sequenced in ONE place because the two are different questions and the order is the answer's quality: a mutant whose own clock says its bound never fired names the cause, and a run past its total budget names only the symptom. #> [OutputType([string])] [CmdletBinding()] param( [Parameter(Mandatory)] [double]$MutantSeconds, [Parameter(Mandatory)] [double]$ElapsedSeconds, [Parameter(Mandatory)] [int]$TimeoutSeconds, [Parameter(Mandatory)] [int]$DeadlineSeconds, [Parameter(Mandatory)] [int]$Index, [Parameter(Mandatory)] [int]$Total ) $stalled = Get-PSMutationStalledFault -MutantSeconds $MutantSeconds -TimeoutSeconds $TimeoutSeconds ` -Index $Index -Total $Total if ($stalled) { return $stalled } return Get-PSMutationOverBudgetFault -ElapsedSeconds $ElapsedSeconds -DeadlineSeconds $DeadlineSeconds ` -Index $Index -Total $Total } function Complete-PSMutationRetirement { <# .SYNOPSIS Record every finished mutant whose turn has come, IN CANDIDATE ORDER. .DESCRIPTION Workers finish out of order -- a killed mutant stops at the first failing test and a survivor runs the whole suite, which is a 6x spread measured on this repo's sibling -- and a report whose row order depended on that would differ between two runs of the same config. So a finished mutant is parked under its candidate index and only recorded once every mutant before it has been. That is one mechanism answering three requirements at once. The report is deterministic, the progress line stays a monotonic [n/total] rather than jumping about, and the partial report an interrupted run writes is a genuine PREFIX of the full one instead of whichever mutants happened to land first. Nothing waits on it: at most one worker-count's worth of mutants can be finished and unrecorded, and they are still finished. #> [OutputType([void])] [CmdletBinding()] param([Parameter(Mandatory)] $Schedule, [Parameter(Mandatory)] $Context) while ($Schedule.Parked.ContainsKey($Schedule.Retired)) { $done = $Schedule.Parked[$Schedule.Retired] $Schedule.Parked.Remove($Schedule.Retired) $Schedule.Retired++ $Context.Sink.Add($done.Row) Write-PSMutationOutput -Quiet:$Context.Quiet -Lines (Get-PSMutationProgressLine -Index $Schedule.Retired ` -Total $Context.Total -Result $done.Row -DisplayFile (Split-Path $done.Row.File -Leaf)) # Not passed -Quiet: a non-interactive host renders no progress at all, so there is # nothing for -Quiet to silence, and a local run keeps the one signal that tells a hung # run from a slow one even when the log is off. Write-PSMutationProgress -Index $Schedule.Retired -Total $Context.Total -Activity 'Evaluating mutants' # Checked AFTER the row is in the sink. A run stopped here has already recorded # everything it finished, so the partial report written on the way out says how far it # got -- which is the difference between a diagnosable stop and the zero-byte report an # overnight hang leaves. $fault = Get-PSMutationLoopFault -MutantSeconds $done.Seconds -TimeoutSeconds $done.TimeoutSeconds ` -ElapsedSeconds $Context.RunClock.Elapsed.TotalSeconds -DeadlineSeconds $Context.DeadlineSeconds ` -Index $Schedule.Retired -Total $Context.Total if ($fault) { throw $fault } } } function Get-PSMutationUsingModuleFault { # Each `using module <path>` in one file that names code inside the sandbox, as a fault line. # A bare name loads an installed module, which no mutant changes, and a path that resolves # outside the sandbox is not code under test either. [OutputType([string])] [CmdletBinding()] param( [Parameter(Mandatory)] $Ast, [Parameter(Mandatory)] [string]$File, [Parameter(Mandatory)] [string]$SandboxRoot ) $own = $SandboxRoot.TrimEnd('\', '/') + [System.IO.Path]::DirectorySeparatorChar foreach ($u in $Ast.UsingStatements) { $name = [string]$u.Name.Value if ($u.UsingStatementKind -ne 'Module' -or $name -notmatch '[\\/]|\.ps[dm]1$') { continue } $target = [System.IO.Path]::GetFullPath($name, (Split-Path -Parent $File)) if ($target.StartsWith($own, [System.StringComparison]::OrdinalIgnoreCase)) { "$(ConvertFrom-PSMutationSandboxPath -Path $File -SandboxRoot $SandboxRoot):$($u.Extent.StartLineNumber) loads the code under test with ``$($u.Extent.Text)``, which the process keeps whatever later changes on disk. Import it with Import-Module, or dot-source it, in a BeforeAll." } } } function Get-PSMutationAddTypeFault { # A fault line when one candidate lies inside one of its file's Add-Type calls -- inclusive at # both edges, because a mutant spanning the whole call still changes what it compiles. [OutputType([string])] [CmdletBinding()] param( [Parameter(Mandatory)] $Candidate, [AllowEmptyCollection()] [object[]]$AddType = @(), [Parameter(Mandatory)] [string]$SandboxRoot ) foreach ($cmd in $AddType) { if ($Candidate.StartOffset -ge $cmd.Extent.StartOffset -and $Candidate.EndOffset -le $cmd.Extent.EndOffset) { "$(ConvertFrom-PSMutationSandboxPath -Path $Candidate.File -SandboxRoot $SandboxRoot):$($Candidate.Line) has a mutant inside an Add-Type call ($($Candidate.Description)), and a compiled type cannot be compiled again in the process. Leave StringLiteral out of 'operators' for this file's run." } } } function Get-PSMutationReloadFault { <# .SYNOPSIS What a run cannot evaluate because PowerShell keeps it for the life of the PROCESS, as one refusal -- or $null (#249). .DESCRIPTION Every mutant runs in this process: the workers are runspaces in it, and the baseline runs in the caller's session in it. A module imported from the sandbox is unloaded before each mutant (see Get-PSMutationWarmPesterScript), which covers Import-Module with or without -Force and a module defining a class. Two things survive even that, measured rather than assumed: * `using module <path>` -- a test still saw the old module after Remove-Module, in a FRESH runspace, and after its own file was touched. Each worker caches whichever version it loads first: the original for worker 0, which the baseline loaded, and some earlier MUTANT for the others. Only a path is refused; a bare name loads an installed module, which no mutant changes. * a mutant inside an `Add-Type` call -- a compiled type cannot be compiled again in the process, so the mutant's Add-Type fails with "already exists" and the old type stays. Only the opt-in StringLiteral operator reaches the source text of such a call. A refusal rather than a quiet guess, because both produce verdicts about code that did not run. Named by file and line, relative to the source root. .OUTPUTS [string] the refusal, or $null. #> [OutputType([string])] [CmdletBinding()] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]]$MutateFiles, [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]]$TestFiles, [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]]$Candidates, [Parameter(Mandatory)] [string]$SandboxRoot ) $found = [System.Collections.Generic.List[string]]::new() $addType = @{} foreach ($file in @(@($MutateFiles) + @($TestFiles) | Select-Object -Unique)) { $ast = [System.Management.Automation.Language.Parser]::ParseFile($file, [ref]$null, [ref]$null) foreach ($f in (Get-PSMutationUsingModuleFault -Ast $ast -File $file -SandboxRoot $SandboxRoot)) { $found.Add($f) } $addType[$file] = @($ast.FindAll({ param($n) $n -is [System.Management.Automation.Language.CommandAst] -and $n.GetCommandName() -eq 'Add-Type' }, $true)) } foreach ($c in $Candidates) { foreach ($f in (Get-PSMutationAddTypeFault -Candidate $c -AddType $addType[$c.File] -SandboxRoot $SandboxRoot)) { $found.Add($f) } } if ($found.Count -eq 0) { return $null } return ("PSMutant evaluates every mutant in this one process, and PowerShell keeps these for its whole life, so their mutants would be judged against code that did not run:`n " + ($found -join "`n ")) } function Get-PSMutationSuiteGroup { <# .SYNOPSIS The distinct covering suites a candidate set will run, each once, in first-use order. Pure. .DESCRIPTION Several mutate files can share one suite, and every candidate of a file shares its file's, so the green check would otherwise run the same tests once per candidate. Keyed on the joined paths rather than on the file, because two files mapped to the same tests are one suite to prove and not two. #> [OutputType([object[]])] [CmdletBinding()] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]]$Candidates, [Parameter(Mandatory)] [hashtable]$TestsByFile, [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]]$AllTests ) # A set, not a hashtable of flags: Add answers whether the key was new, so there is no stored # value nothing reads -- which the mutation gate found as a `$true -> $false` that survived. $seen = [System.Collections.Generic.HashSet[string]]::new() $groups = [System.Collections.Generic.List[object]]::new() foreach ($c in $Candidates) { $suite = Get-PSMutationCoveringSuite -File $c.File -TestsByFile $TestsByFile -AllTests $AllTests if ($seen.Add($suite -join "`n")) { $groups.Add($suite) } } return , $groups.ToArray() } function Get-PSMutationWorkerGreenFault { <# .SYNOPSIS What one unmutated run of a covering suite, in a worker's runspace, says about the run. Pure. .DESCRIPTION $null when the suite passed there. Otherwise the refusal, naming the suite and the worker, because "the suite is not green" is false in the session the user is looking at -- the baseline just said it was -- and the reader needs to know where else it was run. .OUTPUTS [string] the refusal, or $null. #> [OutputType([string])] [CmdletBinding()] param( [Parameter(Mandatory)] $Run, [Parameter(Mandatory)] [string[]]$Suite, [Parameter(Mandatory)] [int]$WorkerId, [Parameter(Mandatory)] [int]$TimeoutSeconds ) if ($Run.Result -eq 'Passed') { return $null } $where = "$($Suite -join ', ') in worker $WorkerId" if ($Run.Result -eq 'TimedOut') { return "The unmutated covering suite did not finish within the per-mutant timeout of $TimeoutSeconds s: $where. Every mutant it covers would time out, and a timeout scores with the kills." } # Named, and capped at ten, as the baseline guard does: a suite that depends on the session # usually fails every test it has, and the first name is the one most likely to explain them. $named = @($Run.Killers) $failed = ' No test carried the error, so a BeforeAll failed: run the file alone in a new session to see it.' if ($named.Count -gt 0) { $more = '' if ($named.Count -gt 10) { $more = " (and $($named.Count - 10) more)" } $failed = " Failed: $(@($named | Select-Object -First 10) -join '; ')$more." } return ("The covering suite passes in this session but not in a clean runspace, which is where every mutant runs: $where.$failed " + 'It depends on something the calling session provides -- a loaded module, a global variable or function, ' + '$ErrorActionPreference, strict mode, $PSDefaultParameterValues -- and every mutant it covers would be scored ' + 'Killed by that failure rather than by a test.') } function Start-PSMutationGreenJob { # One worker's unmutated run of one suite, started and not waited for. Its own function so the # check's loop stays under the cognitive ceiling this repo gates itself on. [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'Starts an in-process pipeline on an unmutated suite; nothing is written.')] [OutputType([pscustomobject])] [CmdletBinding()] param( [Parameter(Mandatory)] [string[]]$Suite, [Parameter(Mandatory)] [int]$WorkerId, [Parameter(Mandatory)] [string]$WorkerRoot, [Parameter(Mandatory)] [string]$SandboxRoot ) # Re-rooted into THIS worker's copy, as a mutant's tests are: the primary sandbox's files # would prove the primary copy green and say nothing about this one. $tests = [string[]]@($Suite | ForEach-Object { Get-PSMutationWorkerPath -Path $_ -SandboxRoot $SandboxRoot -WorkerRoot $WorkerRoot }) # Every failing test is wanted here, not the first: this is a refusal the reader has to act # on, and it is paid once per run rather than once per mutant. $ps = Get-PSMutationSuiteShell -WorkerId $WorkerId -Tests $tests -SandboxRoot $WorkerRoot -RecordAllKillers return [pscustomobject]@{ WorkerId = $WorkerId; Shell = $ps; Async = $ps.BeginInvoke() } } function Assert-PSMutationWorkerGreen { <# .SYNOPSIS Run every covering suite the candidates will use, unmutated, in every worker, and refuse the run if any is not green there (#248). .DESCRIPTION The baseline runs in the CALLER'S session: its loaded modules, its global functions and variables, its $ErrorActionPreference and strict mode. A mutant runs in a worker runspace that has only Pester. A suite green only because of the first is red in the second for a reason that has nothing to do with any mutant, and every mutant is then scored Killed -- a perfect score that nothing flags. Stryker's initial test run exists for the same reason. Through the SAME shell, script and sandbox a mutant uses, so passing here means what a mutant's Passed would mean. Every worker, not one: each has its own sandbox copy, and a suite red in one copy would kill that worker's mutants only. The workers run in parallel, so the cost is about one pass over the covering suites -- not one per worker. Every job is collected before any fault is raised, so no runspace is left mid-run when the refusal unwinds the run. Called by the two callers of Invoke-PSMutationLoop, just before it, rather than from inside it. A throw inside the loop is an interruption to its callers, which write a partial report and say "Interrupted after 0 of N" -- the wrong account of a run that was refused. #> [OutputType([hashtable])] [CmdletBinding()] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]]$Candidates, [Parameter(Mandatory)] [hashtable]$TestsByFile, [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]]$AllTests, [Parameter(Mandatory)] [int]$TimeoutSeconds, [Parameter(Mandatory)] [string]$SandboxRoot, # The extra sandboxes, as Invoke-PSMutationLoop takes them: worker 0 is the primary. [AllowEmptyCollection()] [string[]]$WorkerSandbox = @() ) $roots = [string[]]@(@($SandboxRoot) + @($WorkerSandbox)) $bySuite = @{} foreach ($suite in (Get-PSMutationSuiteGroup -Candidates $Candidates -TestsByFile $TestsByFile -AllTests $AllTests)) { # Timed across ALL workers, so it is the slowest copy's time: the run waits for each job # in turn, and the clock stops after the last. Measured under the same parallel load the # mutants will run under, which is what makes it the right basis for their deadlines. $clock = [System.Diagnostics.Stopwatch]::StartNew() $jobs = @(for ($w = 0; $w -lt $roots.Count; $w++) { Start-PSMutationGreenJob -Suite $suite -WorkerId $w -WorkerRoot $roots[$w] -SandboxRoot $SandboxRoot }) # Named as the report names files, relative to the source root: the sandbox is a temp # directory the reader has never seen, and which copy a worker ran the worker number says. $shown = [string[]]@($suite | ForEach-Object { ConvertFrom-PSMutationSandboxPath -Path $_ -SandboxRoot $SandboxRoot }) $faults = foreach ($job in $jobs) { $done = $job.Async.AsyncWaitHandle.WaitOne([timespan]::FromSeconds($TimeoutSeconds)) $run = $done ? (Receive-PSMutationJob -Job $job) : (Stop-PSMutationJob -Job $job) Get-PSMutationWorkerGreenFault -Run $run -Suite $shown -WorkerId $job.WorkerId -TimeoutSeconds $TimeoutSeconds } # A passing job yields $null, which a foreach statement still collects -- so the first # FAULT is asked for, not the first element. $first = @($faults | Where-Object { $_ })[0] if ($first) { throw $first } $bySuite[$suite -join "`n"] = $clock.Elapsed.TotalSeconds } return Get-PSMutationFileSecond -Candidates $Candidates -TestsByFile $TestsByFile -AllTests $AllTests -SuiteSeconds $bySuite } function Get-PSMutationFileSecond { # Mutate file -> seconds its covering suite took in the unmutated check (#253). Pure. Keyed by # FILE because that is what a candidate carries; several files sharing one suite share its time. [OutputType([hashtable])] [CmdletBinding()] param( [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]]$Candidates, [Parameter(Mandatory)] [hashtable]$TestsByFile, [Parameter(Mandatory)] [AllowEmptyCollection()] [string[]]$AllTests, [Parameter(Mandatory)] [hashtable]$SuiteSeconds ) $out = @{} foreach ($c in $Candidates) { $out[$c.File] = $SuiteSeconds[(Get-PSMutationCoveringSuite -File $c.File -TestsByFile $TestsByFile -AllTests $AllTests) -join "`n"] } return $out } function Invoke-PSMutationLoop { <# .SYNOPSIS Evaluate every candidate across one or more workers; return the result rows. .DESCRIPTION A serial run is a pool of ONE worker through this same scheduler, not a separate simpler route. Two paths would be two places where a verdict is decided, and they would disagree in whichever case nobody tests -- so `workers` changes how many mutants are in flight and nothing else. The report is identical either way, which tests/EndToEnd.Tests.ps1 asserts by running the same fixture both ways and comparing. Each worker owns its own sandbox copy and its own Pester-loaded runspace, so nothing is shared but the read-only candidate list. That is what the isolation rests on: two workers writing the same file would splice one mutant over another and score both against the wrong source. #> [OutputType([object[]])] [CmdletBinding()] param( # Empty is a legitimate input, not an error: a mutate file may contribute no # covered candidates, and a recheck run whose previous survivors are all dead # has nothing left to evaluate. Both should report zero, not throw. [Parameter(Mandatory)] [AllowEmptyCollection()] [object[]]$Candidates, [Parameter(Mandatory)] [hashtable]$TestsByFile, [Parameter(Mandatory)] [string[]]$AllTests, [Parameter(Mandatory)] [int]$TimeoutSeconds, # Each mutate file's own deadline (#253), from its covering suite's time in the unmutated # check. A file not in it gets TimeoutSeconds. [hashtable]$TimeoutByFile = @{}, # The caller's accumulator, so an INTERRUPTED run still has its rows. # # A run is long -- long enough that losing one to Ctrl-C or a cancelled CI job is an # ordinary event rather than an exceptional one -- and every row lived in a local list # that died with the loop. Handing the list in means the caller still holds whatever was # evaluated after the loop stops, however it stopped. # # MANDATORY rather than optional-with-a-fallback. An internal list used when none was # supplied would be a branch whose two arms produce identical output, which no test could # tell from its own absence -- the same reason -UnitTable is mandatory in the sibling. # AllowEmptyCollection because it is ALWAYS empty here: a mandatory parameter refuses an # empty collection, so without this the loop threw on every single call. [Parameter(Mandatory)] [AllowEmptyCollection()] [System.Collections.Generic.List[object]]$Sink, [string]$SandboxRoot, # The EXTRA sandboxes, one per additional worker; worker 0 mutates the primary sandbox. # The worker count is derived from this rather than given beside it, so there is no way # to ask for four workers and hand over two sandboxes. [AllowEmptyCollection()] [string[]]$WorkerSandbox = @(), [switch]$Quiet, [switch]$RecordAllKillers, # Wall-clock budget for the whole loop. Zero disables it. Checked as a mutant is RETIRED, # never inside one, so the mutant it stops after has already had its file restored. Other # workers may still be mid-flight with a file spliced -- their sandboxes are removed by # the run's own finally, which is the same guarantee that covers a Ctrl-C. [int]$DeadlineSeconds = 0 ) $roots = [string[]]@(@($SandboxRoot) + @($WorkerSandbox)) $context = [pscustomobject]@{ Candidates = $Candidates; Total = $Candidates.Count; Roots = $roots; SandboxRoot = $SandboxRoot TestsByFile = $TestsByFile; AllTests = $AllTests; TimeoutSeconds = $TimeoutSeconds; TimeoutByFile = $TimeoutByFile RecordAllKillers = [bool]$RecordAllKillers; Quiet = [bool]$Quiet DeadlineSeconds = $DeadlineSeconds; Originals = @{}; Sink = $Sink RunClock = [System.Diagnostics.Stopwatch]::StartNew() MarkTests = Test-PSMutationDefinesType -Path @($Candidates | ForEach-Object { $_.File } | Select-Object -Unique) } # Mutable, and a pscustomobject rather than a hashtable so the four fields are named at every # use. The counters have to live here rather than in locals: dispatch, sweep and retirement # each move them, and PowerShell would give each function its own copy of a local. $schedule = [pscustomobject]@{ InFlight = [object[]]::new($roots.Count); Parked = @{}; Next = 0; Retired = 0 } while ($schedule.Retired -lt $context.Total) { Start-PSMutationDispatch -Schedule $schedule -Context $context Wait-PSMutationWorker -Schedule $schedule -Context $context Complete-PSMutationSweep -Schedule $schedule -Context $context Complete-PSMutationRetirement -Schedule $schedule -Context $context } return , $Sink.ToArray() } |