Private/Kinds/Stress.ps1
|
# The Stress Kind: what the processor does when it is actually asked to work. # # Every other Check reads a machine at rest, and a throttling laptop at rest looks # identical to a healthy one. This is the Check that tells them apart: it loads every core # for a chosen duration and watches whether the clock falls away, how hot it gets, and # whether anything breaks. A Customer whose machine "gets slow after a while" is describing # this measurement and nothing else in the Report. # # It runs in the Elevated Part so the temperature is readable, which is the difference # between "the clock dropped" and "the clock dropped because it is 97 C in there". $script:StressNeedsAdmin = $true # How often the sampler looks while the load runs. Not a Check Definition parameter, for # the reason in Private/Sampling.ps1: a Customer may disagree about a threshold, not about # how often a measurement is taken. $script:StressSampleIntervalSeconds = 2 # The first few seconds are the machine winding up, and a clock that has not reached its # boost yet would read as throttling. Samples before this are discarded. $script:StressWarmUpSeconds = 3 # Below this many usable samples there is no start and no end to compare, only a reading. $script:StressMinimumSamples = 4 # How long to let the machine cool afterwards, so the Checks that follow are not measuring # a hot machine. The Elevated Part runs before the Main Part's Checks, so without this # every temperature and clock reading after it would be this Check's own fault. $script:StressCoolDownSeconds = 20 function Get-StressData { <# .SYNOPSIS Loads every core and records what happened. Decides nothing about it. .DESCRIPTION Deliberately untested, and recorded as such in #1 and ADR-0002: generating real load on real cores cannot be faked, and a fake would be testing the fake. What is tested is everything below this function. #> [CmdletBinding()] [OutputType([psobject])] param([hashtable]$Parameters = @{}) $duration = [int](Get-Parameter $Parameters 'DurationSeconds' 60) $processors = [Environment]::ProcessorCount $data = [ordered]@{ PSTypeName = 'Gutcheck.Data.Stress' DurationSeconds = $duration LogicalProcessors = $processors Samples = @() TotalOperations = 0 WorkerErrors = 0 HardwareErrorCount = 0 ThrottleEventCount = 0 Elevated = (Get-CurrentPrivilege) -eq 'admin' } if ($duration -le 0) { return [pscustomobject]$data } $startedAt = Get-Date $endAtUtc = [DateTime]::UtcNow.AddSeconds($duration) $work = { param([DateTime]$End) $operations = [long]0 $value = 1.0 while ([DateTime]::UtcNow -lt $End) { for ($i = 0; $i -lt 10000; $i++) { $value = [Math]::Sqrt($value + $i) * 1.0001 } $operations += 10000 } $operations } $runners = @(foreach ($n in 1..$processors) { $shell = [PowerShell]::Create() [void]$shell.AddScript($work.ToString()).AddArgument($endAtUtc) [pscustomobject]@{ Shell = $shell; Handle = $shell.BeginInvoke() } }) $samples = New-Object System.Collections.Generic.List[psobject] $filter = "Name='_Total'" # The first read of a formatted-performance class is primed with zeroes; discard it. [void](Get-CimInstance Win32_PerfFormattedData_Counters_ProcessorInformation -Filter $filter -ErrorAction SilentlyContinue) while ([DateTime]::UtcNow -lt $endAtUtc) { Start-Sleep -Seconds $script:StressSampleIntervalSeconds $counter = Get-CimInstance Win32_PerfFormattedData_Counters_ProcessorInformation -Filter $filter -ErrorAction SilentlyContinue $samples.Add([pscustomobject]@{ Second = [int]((Get-Date) - $startedAt).TotalSeconds LoadPercent = $counter.PercentProcessorTime PerformancePercent = $counter.PercentProcessorPerformance EffectiveMHz = $(if ($counter) { [math]::Round($counter.ProcessorFrequency * $counter.PercentProcessorPerformance / 100) } else { $null }) LimitPercent = $counter.PercentPerformanceLimit TemperatureC = Get-HottestThermalZone }) } $operations = [long]0 $errors = 0 foreach ($runner in $runners) { try { $returned = $runner.Shell.EndInvoke($runner.Handle) if ($runner.Shell.HadErrors) { $errors++ } $operations += [long](@($returned) | Select-Object -Last 1) } catch { $errors++ } finally { $runner.Shell.Dispose() } } # Only events logged during the load count. A WHEA error from last Tuesday belongs to # the Stability Check; one that happened while the cores were hot belongs here. $unreadable = @{} $data.HardwareErrorCount = @(Get-StabilityEvent -Log 'System' ` -Provider @('Microsoft-Windows-WHEA-Logger') -Id @() -Since $startedAt -Unreadable $unreadable).Count $data.ThrottleEventCount = @(Get-StabilityEvent -Log 'System' ` -Provider @('Microsoft-Windows-Kernel-Processor-Power') -Id @(37) -Since $startedAt -Unreadable $unreadable).Count $data.Samples = @($samples) $data.TotalOperations = $operations $data.WorkerErrors = $errors # The machine is hot now, and every Check after this one would measure that rather # than the machine. Scaled by the shared sampling knob so a test can collapse it. Wait-Sample -Seconds $script:StressCoolDownSeconds [pscustomobject]$data } function Get-HottestThermalZone { <# .SYNOPSIS The highest ACPI reading right now, converted but not filtered, or $null for none. .DESCRIPTION Not filtered for plausibility here, which the Temperature Kind spells out and this one has to agree with: dropping a reading moves the peak, the peak decides the Severity, and a Gatherer that could move a Severity is a Gatherer with judgement in it. A firmware answering with something that is not a temperature is the Judge's problem, and it filters the same range on the way through. #> [CmdletBinding()] param() try { $readings = @(Get-CimInstance -Namespace root/wmi -ClassName MSAcpi_ThermalZoneTemperature -ErrorAction Stop | ForEach-Object { [math]::Round(($_.CurrentTemperature / 10 - 273.15), 1) }) if (-not $readings.Count) { return $null } ($readings | Measure-Object -Maximum).Maximum } catch { $null } } function Get-StressWindow { <# .SYNOPSIS The usable samples, and the start and end quarters to compare. Pure. .DESCRIPTION Separated from the Findings because every clock judgement depends on it, and because "did the clock fall away" is a question about two ends of a window rather than about any single reading. #> [CmdletBinding()] [OutputType([psobject])] param([AllowNull()][AllowEmptyCollection()]$Sample) $usable = @(@($Sample) | Where-Object { $_ -and (ConvertTo-Number $_.Second) -gt $script:StressWarmUpSeconds }) if ($usable.Count -lt $script:StressMinimumSamples) { return [pscustomobject]@{ Usable = @($usable); Comparable = $false; First = @(); Last = @() } } $quarter = [math]::Max(1, [int]($usable.Count / 4)) [pscustomobject]@{ Usable = $usable Comparable = $true First = @($usable | Select-Object -First $quarter) Last = @($usable | Select-Object -Last $quarter) } } function ConvertTo-StressFinding { [CmdletBinding()] [OutputType([psobject])] param( [AllowNull()]$Data, [hashtable]$Parameters = @{} ) $window = Get-StressWindow -Sample (Get-DataCollection $Data 'Samples') New-StressLoadFinding -Data $Data -Parameters $Parameters -Window $window New-StressClockFinding -Data $Data -Parameters $Parameters -Window $window New-StressTemperatureFinding -Data $Data -Parameters $Parameters -Window $window New-StressScoreFinding -Data $Data -Parameters $Parameters New-StressStabilityFinding -Data $Data -Parameters $Parameters New-StressThrottleFinding -Data $Data -Parameters $Parameters } function ConvertTo-StressSection { [CmdletBinding()] [OutputType([psobject])] param([AllowNull()]$Data) New-Section -Title (Get-Text 'Title.Stress.StressTestSamples') -Row @( (Get-DataCollection $Data 'Samples') | Select-Object Second, LoadPercent, PerformancePercent, EffectiveMHz, LimitPercent, TemperatureC ) } function New-StressLoadFinding { [CmdletBinding()] param([AllowNull()]$Data, [hashtable]$Parameters, [Parameter(Mandatory)]$Window) $warnBelow = Get-Parameter $Parameters 'StressLoadWarnBelowPercent' 85 $failBelow = Get-Parameter $Parameters 'StressLoadFailBelowPercent' ([double]::MinValue) $average = Get-SampleAverage (@($Window.Usable) | ForEach-Object { $_.LoadPercent }) if ($null -eq $average) { return New-UnavailableFinding -Category CPU -Check (Get-Text 'Check.Stress.StressCPULoadReached') ` -Hint (Get-Text 'Hint.Stress.TheProcessorCountersReportedNothing') } # Failing to load the machine fully is itself the finding: something is holding the # processor back, and everything measured below is measured under less load than asked. New-Finding -Category CPU -Check (Get-Text 'Check.Stress.StressCPULoadReached') ` -Severity (Get-SeverityBelow $average $warnBelow $failBelow) -Value ('{0:N0} %' -f $average) ` -Hint (Get-Text 'Hint.Stress.CouldNotFullyLoadCPU') } function New-StressClockFinding { [CmdletBinding()] param([AllowNull()]$Data, [hashtable]$Parameters, [Parameter(Mandatory)]$Window) $endFail = Get-Parameter $Parameters 'StressClockEndFailBelowPercent' 60 $endWarn = Get-Parameter $Parameters 'StressClockEndWarnBelowPercent' 85 $dropWarn = Get-Parameter $Parameters 'StressClockDropWarnAbovePercent' 25 if (-not $Window.Comparable) { return New-UnavailableFinding -Category CPU -Check (Get-Text 'Check.Stress.StressClockStartEnd') ` -Hint (Get-Text 'Hint.Stress.TheTestWasTooShort') } $startPercent = Get-SampleAverage (@($Window.First) | ForEach-Object { $_.PerformancePercent }) $endPercent = Get-SampleAverage (@($Window.Last) | ForEach-Object { $_.PerformancePercent }) $startMHz = Get-SampleAverage (@($Window.First) | ForEach-Object { $_.EffectiveMHz }) $endMHz = Get-SampleAverage (@($Window.Last) | ForEach-Object { $_.EffectiveMHz }) if ($null -eq $startPercent -or $null -eq $endPercent) { return New-UnavailableFinding -Category CPU -Check (Get-Text 'Check.Stress.StressClockStartEnd') ` -Hint (Get-Text 'Hint.Stress.TheProcessorCountersDidNot') } # Two ways to fail: ending slow, or falling away from where it started. A machine that # begins at 70 % and holds it is a different fault from one that begins at 100 % and # decays to 70 %, and the second is the one that says "thermal". $drop = 0 if ($startPercent -gt 0) { $drop = (1 - $endPercent / $startPercent) * 100 } $severity = Get-WorstSeverity ` (Get-SeverityBelow $endPercent $endWarn $endFail) ` (Get-Severity $drop $dropWarn ([double]::MaxValue)) New-Finding -Category CPU -Check (Get-Text 'Check.Stress.StressClockStartEnd') -Severity $severity ` -Value ((Get-Text 'Value.Stress.ClockChange') -f $startPercent, $endPercent, $startMHz, $endMHz) ` -Hint (Get-Text 'Hint.Stress.ClockFallsUnderLoadThermal') } function New-StressTemperatureFinding { [CmdletBinding()] param([AllowNull()]$Data, [hashtable]$Parameters, [Parameter(Mandatory)]$Window) $warn = Get-Parameter $Parameters 'StressTemperatureWarnAboveC' 89 $fail = Get-Parameter $Parameters 'StressTemperatureFailAboveC' 94 # Filtered here rather than in the Gatherer, and against the same range the Temperature # Kind uses: several firmwares answer with a constant that is not a temperature, and # reading one as the machine's peak would condemn a machine that never got warm. $readings = @(@($Window.Usable) | ForEach-Object { ConvertTo-Number $_.TemperatureC } | Where-Object { $null -ne $_ -and $_ -gt $script:TemperaturePlausibleMinC -and $_ -lt $script:TemperaturePlausibleMaxC }) if (-not $readings.Count) { # Without a temperature the clock Finding above says a machine throttled and # cannot say why, which is the whole reason this Check wants admin rights. return New-Finding -Category CPU -Check (Get-Text 'Check.Stress.StressTemperature') -Severity INFO ` -Value $(if (Get-DataProperty $Data 'Elevated') { (Get-Text 'Value.Temperature.NoAcpiThermalZone') } else { (Get-Text 'Value.Temperature.NotRecordedNeedsAdmin') }) ` -Hint (Get-Text 'Hint.Stress.WithoutATemperatureAClock') } $peak = ($readings | Measure-Object -Maximum).Maximum New-Finding -Category CPU -Check (Get-Text 'Check.Stress.StressMaxACPITemperature') ` -Severity (Get-Severity $peak $warn $fail) -Value ("$peak C") ` -Hint (Get-Text 'Hint.Stress.OverheatingCleanTheCoolingSystem') } function New-StressScoreFinding { [CmdletBinding()] param([AllowNull()]$Data, [hashtable]$Parameters) $operations = ConvertTo-Number (Get-DataProperty $Data 'TotalOperations') $duration = ConvertTo-Number (Get-DataProperty $Data 'DurationSeconds') if ($null -eq $operations -or $null -eq $duration -or $duration -le 0) { return New-UnavailableFinding -Category CPU -Check (Get-Text 'Check.Stress.StressBenchmarkScore') ` -Hint (Get-Text 'Hint.Stress.NoWorkWasCompletedSo') } # Never graded. The number means nothing on its own and everything beside the same # model that a Customer says is fine. New-Finding -Category CPU -Check (Get-Text 'Check.Stress.StressBenchmarkScore') -Severity INFO ` -Value ('{0:N1} Mops/s' -f ($operations / $duration / 1e6)) ` -Hint (Get-Text 'Hint.Stress.CompareWithAnIdenticalModel') } function New-StressStabilityFinding { [CmdletBinding()] param([AllowNull()]$Data, [hashtable]$Parameters) $errors = ConvertTo-Number (Get-DataProperty $Data 'WorkerErrors') $hardware = ConvertTo-Number (Get-DataProperty $Data 'HardwareErrorCount') if ($null -eq $errors) { $errors = 0 } if ($null -eq $hardware) { $hardware = 0 } if ($errors -eq 0 -and $hardware -eq 0) { return New-Finding -Category CPU -Check (Get-Text 'Check.Stress.StressStability') -Severity OK ` -Value (Get-Text 'Value.Stress.NoErrorsUnderLoad') } # Anything failing under sustained load is hardware until proven otherwise. This is # the Finding that turns "it crashes sometimes" into a warranty call. New-Finding -Category CPU -Check (Get-Text 'Check.Stress.StressStability') -Severity FAIL ` -Value ('{0} worker errors, {1} WHEA hardware events' -f $errors, $hardware) ` -Hint (Get-Text 'Hint.Stress.ErrorsUnderLoadPointTo') } function New-StressThrottleFinding { [CmdletBinding()] param([AllowNull()]$Data, [hashtable]$Parameters) $warnAbove = Get-Parameter $Parameters 'StressThrottleEventWarnAbove' 0 $count = ConvertTo-Number (Get-DataProperty $Data 'ThrottleEventCount') if ($null -eq $count -or $count -le 0) { return } New-Finding -Category CPU -Check (Get-Text 'Check.Stress.StressFirmwareThrottlingEvents') ` -Severity (Get-Severity $count $warnAbove ([double]::MaxValue)) -Value $count ` -Hint (Get-Text 'Hint.Stress.FirmwareLimitedTheCPUDuring') } |