benchmark/BenchmarkHelpers.ps1
|
function Measure-OctaMetricSamples { <# FR-020/FR-021: every metric is sampled multiple times and cross-validated against an independent measurement source before it is allowed into the report. #> [CmdletBinding()] param( [Parameter(Mandatory)][string]$Name, [Parameter(Mandatory)][scriptblock]$Primary, [Parameter(Mandatory)][scriptblock]$CrossCheck, [double]$TolerancePercent = 10, [int]$Samples = 5 ) $primaryValues = @() $crossValues = @() $crossCheckFailed = $false for ($i = 0; $i -lt $Samples; $i++) { $primaryValues += & $Primary try { $crossValues += & $CrossCheck } catch { # Some systems (locked-down environments, certain Server Core installs) disable # performance counters entirely - treat as "could not cross-validate" rather than # letting the error spam the console once per sample. $crossCheckFailed = $true } Start-Sleep -Milliseconds 200 } $mean = ($primaryValues | Measure-Object -Average).Average $variance = (($primaryValues | ForEach-Object { [Math]::Pow($_ - $mean, 2) }) | Measure-Object -Sum).Sum / $Samples $stdDev = [Math]::Sqrt($variance) if ($crossCheckFailed -or $crossValues.Count -eq 0) { $crossMean = $null $agrees = $false } else { $crossMean = ($crossValues | Measure-Object -Average).Average if ($mean -eq 0 -and $crossMean -eq 0) { $agrees = $true } elseif ($mean -ne 0) { $percentDiff = [Math]::Abs($mean - $crossMean) / [Math]::Abs($mean) * 100 $agrees = $percentDiff -le $TolerancePercent } else { $agrees = $false } } return [pscustomobject]@{ Metric = $Name SampleCount = $Samples Mean = [Math]::Round($mean, 2) StdDev = [Math]::Round($stdDev, 2) CrossCheckMean = if ($null -eq $crossMean) { $null } else { [Math]::Round($crossMean, 2) } CrossCheckAgreedWithinTolerance = $agrees } } |