Private/AzStackHci.VMCheckpointHealth.Assessment.ps1
|
Set-StrictMode -Version 1.0 function Get-HyperVEventPolicy { [OutputType([pscustomobject])] param() [pscustomobject]@{ CriticalIds = @(3216) OperationFailureIds = @(18012, 19100, 16300) LowSignalIds = @(3280, 12240, 15268, 19090, 32510) ContextIds = @(18500, 18510, 19070, 19080) MergeFailureIds = @(19090, 19100, 32510) MergeSuccessIds = @(19080) ForkCommitHResults = @('0x80048102', '0x800703EE') LeadingHResults = @('0x800480BD', '0x800480BC') SymptomHResults = @('0x80070020', '0x80070002') } } function Get-HyperVEventSignalAssessment { [OutputType([pscustomobject])] param( [Parameter(Mandatory)][int]$EventId, [AllowEmptyString()][string]$Log, [AllowEmptyString()][string]$Message, [Parameter(Mandatory)]$Policy ) $hasCheckpointContext = ($Message -match '(?i)checkpoint|differencing|fork|virtual\s+hard\s+disk|\bvhdx?\b') $hasCommitForkError = ($Message -match [regex]::Escape('0x80048102')) $hasFileInvalid = ($Message -match [regex]::Escape('0x800703EE')) $isContextual3216 = (($EventId -eq 3216) -and ($Log -eq 'Worker') -and $hasCheckpointContext) $isConfirming = ($hasCommitForkError -or $isContextual3216 -or ($hasFileInvalid -and $hasCheckpointContext)) $hasLeadingCode = @($Policy.LeadingHResults | Where-Object { $Message -match [regex]::Escape($_) }).Count -gt 0 $hasSymptomCode = @($Policy.SymptomHResults | Where-Object { $Message -match [regex]::Escape($_) }).Count -gt 0 $role = if ($isConfirming) { 'Confirming' } elseif ($hasLeadingCode) { 'Leading' } elseif (($Policy.OperationFailureIds -contains $EventId) -or ($Policy.LowSignalIds -contains $EventId) -or $hasSymptomCode) { 'Operational' } elseif ($Policy.ContextIds -contains $EventId) { 'Context' } else { 'Other' } [pscustomobject]@{ Role = $role; IsConfirmingFork = [bool]$isConfirming; HasCheckpointContext = [bool]$hasCheckpointContext } } function Resolve-HyperVOperationRecovery { [OutputType([pscustomobject])] param( [object[]]$Events = @(), [int[]]$FailureIds = @(18012, 19100, 16300), [int[]]$CompletionIds = @(19080), [int[]]$CompletionEligibleFailureIds = @(19100), [ValidateRange(1, 1440)][int]$MaxMinutes = 30 ) $failures = @($Events | Where-Object { $FailureIds -contains [int]$_.Id } | Sort-Object 'Time (UTC)') $completions = @($Events | Where-Object { $CompletionIds -contains [int]$_.Id } | Sort-Object 'Time (UTC)') if ($failures.Count -eq 0) { return [pscustomobject]@{ Status = 'NotApplicable'; FailureCount = 0; CompletionCount = $completions.Count; CausalMatchCount = 0; ApparentMatchCount = 0; UnresolvedCount = 0 } } $causalMatchCount = 0 $apparentMatchCount = 0 $unresolvedCount = 0 $evidencePattern = '(?i)(?:[a-z]:\\[^\r\n|"''<>]+?\.(?:avhdx|vhdx|vhd)|(?<![0-9a-f])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?![0-9a-f]))' foreach ($failure in $failures) { if ($CompletionEligibleFailureIds -notcontains [int]$failure.Id) { $unresolvedCount++; continue } try { $failureTime = [datetime]::ParseExact([string]$failure.'Time (UTC)', 'yyyy-MM-dd HH:mm:ss', [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal) } catch { $unresolvedCount++; continue } $boundedCompletions = @($completions | Where-Object { try { $completionTime = [datetime]::ParseExact([string]$_.'Time (UTC)', 'yyyy-MM-dd HH:mm:ss', [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::AssumeUniversal) ($completionTime -ge $failureTime) -and ($completionTime -le $failureTime.AddMinutes($MaxMinutes)) } catch { $false } }) if ($boundedCompletions.Count -eq 0) { $unresolvedCount++; continue } $failureKeys = @([regex]::Matches([string]$failure.FullMessage, $evidencePattern) | ForEach-Object { $_.Value.ToLowerInvariant() } | Sort-Object -Unique) $causalCompletion = @($boundedCompletions | Where-Object { $completionKeys = @([regex]::Matches([string]$_.FullMessage, $evidencePattern) | ForEach-Object { $_.Value.ToLowerInvariant() } | Sort-Object -Unique) @($failureKeys | Where-Object { $completionKeys -contains $_ }).Count -gt 0 } | Select-Object -First 1) if ($failureKeys.Count -gt 0 -and $causalCompletion.Count -gt 0) { $causalMatchCount++ } else { $apparentMatchCount++ } } $status = if ($unresolvedCount -gt 0) { 'Unresolved' } elseif ($causalMatchCount -eq $failures.Count) { 'ConfirmedRecovered' } else { 'ApparentlyRecovered' } [pscustomobject]@{ Status = $status; FailureCount = $failures.Count; CompletionCount = $completions.Count; CausalMatchCount = $causalMatchCount; ApparentMatchCount = $apparentMatchCount; UnresolvedCount = $unresolvedCount } } function Get-HyperVEventCsvDisposition { [OutputType([pscustomobject])] param( [Parameter(Mandatory)][object]$Event, [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$Events, [Parameter(Mandatory)][object]$Policy, [AllowEmptyCollection()][object[]]$CompletionEvents ) $eventId = [int]$Event.Id $signalRole = if ($Event.PSObject.Properties['SignalRole']) { [string]$Event.SignalRole } else { '' } $isConfirmingFork = [bool]($Event.PSObject.Properties['IsConfirmingFork'] -and $Event.IsConfirmingFork) if ($isConfirmingFork -or $signalRole -eq 'Confirming') { return [pscustomobject]@{ EventClassification = 'High-signal'; VerdictDriver = $true; RecoveryDisposition = 'Unresolved' DispositionReason = 'Confirming checkpoint fork-commit or rollback evidence contributes to the VM verdict.' } } if ($Policy.OperationFailureIds -contains $eventId) { if (-not $PSBoundParameters.ContainsKey('CompletionEvents')) { $CompletionEvents = @($Events | Where-Object { $Policy.MergeSuccessIds -contains [int]$_.Id }) } $recovery = Resolve-HyperVOperationRecovery -Events (@($Event) + $completionEvents) ` -FailureIds $Policy.OperationFailureIds -CompletionIds $Policy.MergeSuccessIds $verdictDriver = ($recovery.Status -eq 'Unresolved') return [pscustomobject]@{ EventClassification = 'High-signal'; VerdictDriver = [bool]$verdictDriver RecoveryDisposition = [string]$recovery.Status DispositionReason = if ($verdictDriver) { 'The VM-attributed operation failure has no eligible bounded recovery evidence and contributes to the VM verdict.' } else { 'Bounded merge-completion evidence reduces this operation failure to recovered context.' } } } if ($Policy.LowSignalIds -contains $eventId) { return [pscustomobject]@{ EventClassification = 'Low-signal'; VerdictDriver = $false; RecoveryDisposition = 'ContextOnly' DispositionReason = 'The event is retained as low-signal operational context and does not drive the VM verdict by itself.' } } if (($Policy.ContextIds -contains $eventId) -or ($Policy.MergeSuccessIds -contains $eventId) -or $signalRole -eq 'Context') { return [pscustomobject]@{ EventClassification = 'Corroborating'; VerdictDriver = $false; RecoveryDisposition = 'ContextOnly' DispositionReason = 'The event is retained as lifecycle or recovery context and does not drive the VM verdict.' } } [pscustomobject]@{ EventClassification = 'Informational'; VerdictDriver = $false; RecoveryDisposition = 'NotApplicable' DispositionReason = 'The event is informational and has no checkpoint-operation recovery disposition.' } } function ConvertTo-HyperVEventCsvRows { [OutputType([object[]])] param( [AllowEmptyCollection()][object[]]$Events = @(), [Parameter(Mandatory)][object]$Policy, [AllowEmptyString()][string]$VMName, [AllowEmptyString()][string]$VMId, [AllowEmptyString()][string]$DefaultNode, [ValidateSet('StandardLookback', 'HistoricOrphanWindow', 'HistoricActiveCheckpointWindow')] [string]$DefaultEvidenceScope = 'StandardLookback' ) $projected = [System.Collections.Generic.List[object]]::new() $seen = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) foreach ($eventRow in @($Events)) { if (-not $eventRow) { continue } $timeUtc = if ($eventRow.PSObject.Properties['Time (UTC)']) { [string]$eventRow.'Time (UTC)' } elseif ($eventRow.PSObject.Properties['Time']) { [string]$eventRow.Time } else { '' } $node = if ($eventRow.PSObject.Properties['Node'] -and $eventRow.Node) { [string]$eventRow.Node } else { $DefaultNode } $recordId = if ($eventRow.PSObject.Properties['RecordId']) { [long]$eventRow.RecordId } else { 0L } $fullMessage = if ($eventRow.PSObject.Properties['FullMessage']) { [string]$eventRow.FullMessage } elseif ($eventRow.PSObject.Properties['Message']) { [string]$eventRow.Message } else { '' } $identityKey = if ($recordId -gt 0) { '{0}|{1}|{2}' -f $node, [string]$eventRow.Log, $recordId } else { '{0}|{1}|{2}|{3}|{4}' -f $node, [string]$eventRow.Log, $timeUtc, [int]$eventRow.Id, $fullMessage } if (-not $seen.Add($identityKey)) { continue } $attribution = if ($eventRow.PSObject.Properties['VmAttributed']) { [pscustomobject]@{ Attributed = [bool]$eventRow.VmAttributed Method = if ($eventRow.PSObject.Properties['AttributionMethod']) { [string]$eventRow.AttributionMethod } else { 'ExistingAssessment' } Confidence = if ($eventRow.PSObject.Properties['AttributionConfidence']) { [string]$eventRow.AttributionConfidence } else { 'Unknown' } } } else { Resolve-HyperVEventAttribution -Message $fullMessage -VMName $VMName -VMId $VMId } $signal = Get-HyperVEventSignalAssessment -EventId ([int]$eventRow.Id) -Log ([string]$eventRow.Log) -Message $fullMessage -Policy $Policy $concern = if ($eventRow.PSObject.Properties['Concern']) { [string]$eventRow.Concern } elseif ($signal.Role -in @('Confirming', 'Leading', 'Operational')) { 'YES' } else { '' } $scope = if ($eventRow.PSObject.Properties['EvidenceScope'] -and $eventRow.EvidenceScope) { [string]$eventRow.EvidenceScope } else { $DefaultEvidenceScope } $row = [pscustomobject][ordered]@{ 'Time (UTC)' = $timeUtc AuditedVMName = $VMName AuditedVMId = $VMId Node = $node RecordId = $recordId Id = [int]$eventRow.Id Level = if ($eventRow.PSObject.Properties['Level']) { [string]$eventRow.Level } else { '' } Log = [string]$eventRow.Log Concern = $concern CollectedAsConcern = ($concern -eq 'YES') VmAttributed = [bool]$attribution.Attributed AttributionMethod = [string]$attribution.Method AttributionConfidence = [string]$attribution.Confidence EvidenceScope = $scope CorrelationAnchor = if ($eventRow.PSObject.Properties['CorrelationAnchor']) { [string]$eventRow.CorrelationAnchor } else { '' } CorrelationWindowStartUtc = if ($eventRow.PSObject.Properties['CorrelationWindowStartUtc']) { [string]$eventRow.CorrelationWindowStartUtc } else { '' } CorrelationWindowEndUtc = if ($eventRow.PSObject.Properties['CorrelationWindowEndUtc']) { [string]$eventRow.CorrelationWindowEndUtc } else { '' } EventClassification = '' VerdictDriver = $false IsConfirmingFork = [bool]$signal.IsConfirmingFork RecoveryDisposition = '' DispositionReason = '' FullMessage = $fullMessage } $hasDisposition = $eventRow.PSObject.Properties['EventClassification'] -and $eventRow.EventClassification -and $eventRow.PSObject.Properties['RecoveryDisposition'] -and $eventRow.RecoveryDisposition if ($hasDisposition) { $row.EventClassification = [string]$eventRow.EventClassification $row.VerdictDriver = [bool]$eventRow.VerdictDriver $row.RecoveryDisposition = [string]$eventRow.RecoveryDisposition $row.DispositionReason = [string]$eventRow.DispositionReason } else { $disposition = Get-HyperVEventCsvDisposition -Event $row -Events $Events -Policy $Policy $row.EventClassification = [string]$disposition.EventClassification $row.VerdictDriver = [bool]$disposition.VerdictDriver $row.RecoveryDisposition = [string]$disposition.RecoveryDisposition $row.DispositionReason = [string]$disposition.DispositionReason } [void]$projected.Add($row) } $projected.ToArray() } function Compare-VMCollectionStateToken { [OutputType([pscustomobject])] param([Parameter(Mandatory)]$StartToken, [Parameter(Mandatory)]$EndToken) $reasons = [System.Collections.Generic.List[string]]::new() if (-not ([string]$StartToken.OwnerNode).Equals([string]$EndToken.OwnerNode, [StringComparison]::OrdinalIgnoreCase)) { [void]$reasons.Add('OwnerNode') } if (-not ([string]$StartToken.State).Equals([string]$EndToken.State, [StringComparison]::OrdinalIgnoreCase)) { [void]$reasons.Add('State') } if ([int]$StartToken.CheckpointCount -ne [int]$EndToken.CheckpointCount) { [void]$reasons.Add('CheckpointCount') } $startPaths = @($StartToken.DiskPaths | ForEach-Object { ([string]$_).ToLowerInvariant() } | Sort-Object -Unique) $endPaths = @($EndToken.DiskPaths | ForEach-Object { ([string]$_).ToLowerInvariant() } | Sort-Object -Unique) if (($startPaths -join "`n") -ne ($endPaths -join "`n")) { [void]$reasons.Add('DiskPaths') } if ([string]$StartToken.ConfigLastWriteUtc -ne [string]$EndToken.ConfigLastWriteUtc) { [void]$reasons.Add('ConfigLastWriteUtc') } [pscustomobject]@{ Changed = ($reasons.Count -gt 0); Reasons = $reasons.ToArray() } } function Get-VMCollectionStateImpact { [OutputType([string])] param( [Parameter(Mandatory)][ValidateSet('Stable', 'Changed', 'Unavailable')][string]$Status, [string[]]$Reasons = @(), [bool]$ReplicationEnabled = $false, [AllowEmptyString()][string]$ReplicaProductSeverity, [AllowEmptyString()][string]$ReplicaState ) if ($Status -eq 'Stable') { return 'Stable' } $healthyReplicaConfigWrite = ($Status -eq 'Changed' -and @($Reasons).Count -eq 1 -and $Reasons[0] -eq 'ConfigLastWriteUtc' -and $ReplicationEnabled -and $ReplicaProductSeverity -eq 'Healthy' -and $ReplicaState.Equals('Replicating', [StringComparison]::OrdinalIgnoreCase)) if ($healthyReplicaConfigWrite) { return 'Advisory' } 'Inconclusive' } function Get-HyperVReplicationAssessment { [OutputType([pscustomobject])] param( [Parameter(Mandatory)][bool]$Enabled, [AllowEmptyString()][string]$State, [AllowEmptyString()][string]$Health, [AllowEmptyString()][string]$Mode, [bool]$MeasurementsAvailable = $false, [datetime]$LastReplicationTimeUtc = [datetime]::MinValue, [long]$PendingBytes = 0, [double]$LatencySeconds = 0, [long]$MissedCount = 0, [double]$FrequencySeconds = 0, [long]$AverageReplicationBytes = 0, [long]$SuccessfulCount = -1, [double]$MonitoringIntervalSeconds = 0, [datetime]$NowUtc = [datetime]::UtcNow, [int]$MaxAgeMinutes = 60, [long]$MaxPendingMB = 1024, [int]$MaxLatencySeconds = 300, [int]$MaxMissedCount = 0, [double]$MaxAgeCycles = 12, [double]$MaxPendingCycles = 2, [double]$MaxLatencyCycles = 2, [double]$MaxMissedRatePercent = 10, [long]$MinMissedCountForConcern = 3 ) if (-not $Enabled) { return [pscustomobject]@{ Severity = 'NotApplicable'; ProductSeverity = 'NotApplicable'; MeasurementStatus = 'NotApplicable' IsConcern = $false; HasAdvisory = $false; IsCritical = $false State = $State; Health = $Health; Mode = $Mode; Reason = 'Hyper-V Replica is disabled.' ThresholdBreaches = @(); ConcernBreaches = @(); AdvisoryBreaches = @(); MeasurementsAvailable = $false } } $normalizedHealth = $Health.Trim() $normalizedState = $State.Trim() $productSeverity = switch ($normalizedHealth.ToLowerInvariant()) { 'critical' { 'Critical'; break } 'warning' { 'Warning'; break } 'normal' { if ($normalizedState) { 'Healthy' } else { 'Unknown' }; break } default { 'Unknown' } } $thresholdBreaches = [System.Collections.Generic.List[string]]::new() $concernBreaches = [System.Collections.Generic.List[string]]::new() $advisoryBreaches = [System.Collections.Generic.List[string]]::new() $effectiveAgeMinutes = [double]$MaxAgeMinutes $effectivePendingBytes = [long]($MaxPendingMB * 1MB) $effectiveLatencySeconds = [double]$MaxLatencySeconds if ($FrequencySeconds -gt 0) { $effectiveAgeMinutes = [math]::Max($effectiveAgeMinutes, (($FrequencySeconds * $MaxAgeCycles) / 60.0)) $effectiveLatencySeconds = [math]::Max($effectiveLatencySeconds, ($FrequencySeconds * $MaxLatencyCycles)) } if ($AverageReplicationBytes -gt 0) { $relativePendingBytes = [double]$AverageReplicationBytes * $MaxPendingCycles if ($relativePendingBytes -gt $effectivePendingBytes) { $effectivePendingBytes = [long][math]::Ceiling($relativePendingBytes) } } $lastReplicationAgeMinutes = $null $missedRatePercent = $null if ($MeasurementsAvailable) { $totalMeasuredCount = $SuccessfulCount + $MissedCount if ($SuccessfulCount -ge 0 -and $totalMeasuredCount -gt 0) { $missedRatePercent = (100.0 * $MissedCount) / $totalMeasuredCount } if ($LastReplicationTimeUtc -ne [datetime]::MinValue) { $lastReplicationAgeMinutes = ($NowUtc.ToUniversalTime() - $LastReplicationTimeUtc.ToUniversalTime()).TotalMinutes if ($lastReplicationAgeMinutes -gt $MaxAgeMinutes) { [void]$thresholdBreaches.Add('LastReplicationAge') if ($lastReplicationAgeMinutes -gt $effectiveAgeMinutes) { [void]$concernBreaches.Add('LastReplicationAge') } else { [void]$advisoryBreaches.Add('LastReplicationAge') } } } if ($PendingBytes -gt ($MaxPendingMB * 1MB)) { [void]$thresholdBreaches.Add('PendingBytes') if ($PendingBytes -gt $effectivePendingBytes) { [void]$concernBreaches.Add('PendingBytes') } else { [void]$advisoryBreaches.Add('PendingBytes') } } if ($LatencySeconds -gt $MaxLatencySeconds) { [void]$thresholdBreaches.Add('Latency') if ($LatencySeconds -gt $effectiveLatencySeconds) { [void]$concernBreaches.Add('Latency') } else { [void]$advisoryBreaches.Add('Latency') } } if ($MissedCount -gt $MaxMissedCount) { [void]$thresholdBreaches.Add('MissedCount') $missedIsConcern = ($MissedCount -ge $MinMissedCountForConcern) -and (($null -eq $missedRatePercent) -or ($missedRatePercent -gt $MaxMissedRatePercent)) if ($missedIsConcern) { [void]$concernBreaches.Add('MissedCount') } else { [void]$advisoryBreaches.Add('MissedCount') } } } $measurementStatus = if ($concernBreaches.Count -gt 0) { 'Concern' } elseif ($advisoryBreaches.Count -gt 0) { 'Advisory' } elseif ($MeasurementsAvailable) { 'Healthy' } else { 'Unavailable' } $severity = if ($productSeverity -eq 'Healthy' -and $measurementStatus -eq 'Concern') { 'Warning' } else { $productSeverity } $isConcern = ($productSeverity -in @('Critical', 'Warning', 'Unknown')) -or ($measurementStatus -eq 'Concern') [pscustomobject]@{ Severity = $severity; ProductSeverity = $productSeverity; MeasurementStatus = $measurementStatus IsConcern = $isConcern; HasAdvisory = ($measurementStatus -eq 'Advisory'); IsCritical = ($productSeverity -eq 'Critical') State = $normalizedState; Health = $normalizedHealth; Mode = $Mode.Trim() MeasurementsAvailable = $MeasurementsAvailable; LastReplicationTimeUtc = $LastReplicationTimeUtc LastReplicationAgeMinutes = $lastReplicationAgeMinutes PendingBytes = $PendingBytes; LatencySeconds = $LatencySeconds; MissedCount = $MissedCount FrequencySeconds = $FrequencySeconds; AverageReplicationBytes = $AverageReplicationBytes SuccessfulCount = $SuccessfulCount; MissedRatePercent = $missedRatePercent MonitoringIntervalSeconds = $MonitoringIntervalSeconds EffectiveMaxAgeMinutes = $effectiveAgeMinutes; EffectiveMaxPendingBytes = $effectivePendingBytes EffectiveMaxLatencySeconds = $effectiveLatencySeconds; MaxMissedRatePercent = $MaxMissedRatePercent ThresholdBreaches = $thresholdBreaches.ToArray() ConcernBreaches = $concernBreaches.ToArray(); AdvisoryBreaches = $advisoryBreaches.ToArray() Reason = if ($productSeverity -eq 'Critical') { 'Hyper-V Replica health is Critical.' } elseif ($productSeverity -eq 'Warning') { 'Hyper-V Replica health is Warning.' } elseif ($productSeverity -eq 'Unknown') { 'Hyper-V Replica is enabled but health or state evidence is unavailable.' } elseif ($measurementStatus -eq 'Concern') { "Hyper-V Replica measurements significantly exceed the limits calculated for this VM's replication frequency." } elseif ($measurementStatus -eq 'Advisory') { 'One Hyper-V Replica measurement is outside its expected range while product health remains Normal.' } else { 'Hyper-V Replica reports Normal health with an available state.' } } } function Resolve-HyperVEventAttribution { [OutputType([pscustomobject])] param([AllowEmptyString()][string]$Message, [AllowEmptyString()][string]$VMName, [AllowEmptyString()][string]$VMId) $normalizedTargetId = $VMId.Trim().Trim('{', '}', '(', ')') $guidPattern = '(?i)(?<![0-9a-f])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?![0-9a-f])' $guidMatches = [regex]::Matches($Message, $guidPattern) if ($guidMatches.Count -gt 0) { $attributed = $false foreach ($guidMatch in $guidMatches) { if ($normalizedTargetId -and $guidMatch.Value.Equals($normalizedTargetId, [StringComparison]::OrdinalIgnoreCase)) { $attributed = $true; break } } return [pscustomobject]@{ Attributed = $attributed; Method = 'StructuredGuid'; Confidence = 'High'; StructuredIdentifierPresent = $true } } $namePattern = '(?i)\b(?:virtual\s+machine|vm)\s+(?:name\s*[:=]?\s*)?[''\"](?<Name>[^''\"]+)[''\"]' $nameMatches = [regex]::Matches($Message, $namePattern) if ($nameMatches.Count -gt 0) { $attributed = $false foreach ($nameMatch in $nameMatches) { if ($VMName -and $nameMatch.Groups['Name'].Value.Equals($VMName, [StringComparison]::OrdinalIgnoreCase)) { $attributed = $true; break } } return [pscustomobject]@{ Attributed = $attributed; Method = 'StructuredName'; Confidence = 'High'; StructuredIdentifierPresent = $true } } $fallbackAttributed = $false if ($VMName) { $boundedNamePattern = '(?i)(?<![\p{L}\p{N}_\\/-])' + [regex]::Escape($VMName) + '(?![\p{L}\p{N}_\\/-])'; $fallbackAttributed = [regex]::IsMatch($Message, $boundedNamePattern) } [pscustomobject]@{ Attributed = $fallbackAttributed; Method = 'BoundedNameFallback'; Confidence = if ($fallbackAttributed) { 'Low' } else { 'None' }; StructuredIdentifierPresent = $false } } function New-HyperVEventIdentityIndex { [OutputType([pscustomobject])] param([AllowEmptyCollection()][object[]]$Events = @()) $byGuid = @{} $byName = @{} $unstructured = [System.Collections.Generic.List[object]]::new() $guidPattern = '(?i)(?<![0-9a-f])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?![0-9a-f])' $namePattern = '(?i)\b(?:virtual\s+machine|vm)\s+(?:name\s*[:=]?\s*)?[''\"](?<Name>[^''\"]+)[''\"]' foreach ($eventRow in @($Events)) { if (-not $eventRow) { continue } $message = if ($eventRow.PSObject.Properties['FullMessage']) { [string]$eventRow.FullMessage } else { [string]$eventRow.Message } $guidKeys = @([regex]::Matches($message, $guidPattern) | ForEach-Object { $_.Value.ToLowerInvariant() } | Sort-Object -Unique) if ($guidKeys.Count -gt 0) { foreach ($key in $guidKeys) { if (-not $byGuid.ContainsKey($key)) { $byGuid[$key] = [System.Collections.Generic.List[object]]::new() } [void]$byGuid[$key].Add($eventRow) } continue } $nameKeys = @([regex]::Matches($message, $namePattern) | ForEach-Object { $_.Groups['Name'].Value.ToLowerInvariant() } | Sort-Object -Unique) if ($nameKeys.Count -gt 0) { foreach ($key in $nameKeys) { if (-not $byName.ContainsKey($key)) { $byName[$key] = [System.Collections.Generic.List[object]]::new() } [void]$byName[$key].Add($eventRow) } continue } [void]$unstructured.Add($eventRow) } [pscustomobject]@{ ByGuid = $byGuid; ByName = $byName; Unstructured = $unstructured.ToArray(); TotalCount = @($Events).Count } } function Select-HyperVEventsForVM { [OutputType([object[]])] param( [Parameter(Mandatory)][object]$Index, [AllowEmptyString()][string]$VMName, [AllowEmptyString()][string]$VMId ) $selected = [System.Collections.Generic.List[object]]::new() $normalizedId = $VMId.Trim().Trim('{', '}', '(', ')').ToLowerInvariant() $normalizedName = $VMName.ToLowerInvariant() if ($normalizedId -and $Index.ByGuid.ContainsKey($normalizedId)) { foreach ($eventRow in $Index.ByGuid[$normalizedId]) { [void]$selected.Add($eventRow) } } if ($normalizedName -and $Index.ByName.ContainsKey($normalizedName)) { foreach ($eventRow in $Index.ByName[$normalizedName]) { [void]$selected.Add($eventRow) } } foreach ($eventRow in @($Index.Unstructured)) { $message = if ($eventRow.PSObject.Properties['FullMessage']) { [string]$eventRow.FullMessage } else { [string]$eventRow.Message } $attribution = Resolve-HyperVEventAttribution -Message $message -VMName $VMName -VMId $VMId if ($attribution.Attributed) { [void]$selected.Add($eventRow) } } $selected.ToArray() } function Get-ClusterRoleVMAbsenceAssessment { [OutputType([pscustomobject])] param( [Parameter(Mandatory)][ValidateNotNullOrEmpty()][string]$RoleOwner, [AllowEmptyString()][string]$FoundNode, [ValidateRange(0, 1024)][int]$FailedNodeCount = 0 ) if ($FoundNode) { return [pscustomobject]@{ Category = 'OwnerMismatch' Detail = "The cluster role records owner '$RoleOwner', but the Hyper-V VM was found on '$FoundNode'. The role ownership and Hyper-V inventory are inconsistent; verify the clustered role and VM resources." } } if ($FailedNodeCount -gt 0) { return [pscustomobject]@{ Category = 'VerificationIncomplete' Detail = "The cluster role exists and records owner '$RoleOwner', but the Hyper-V VM was not found there. Cluster-wide verification was incomplete because $FailedNodeCount node(s) could not be queried." } } [pscustomobject]@{ Category = 'StaleClusterRoleCandidate' Detail = "The cluster role exists and records owner '$RoleOwner', but no Hyper-V VM with this name was found on any cluster node. This can occur when a VM is deleted in Hyper-V but its Failover Clustering role is not removed; verify the role and its resources." } } function Resolve-EventCoverage { [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)][AllowEmptyCollection()][object[]]$CoverageRows, [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$ExpectedNodes, [Parameter(Mandatory)][AllowEmptyCollection()][string[]]$ExpectedChannels, [Parameter(Mandatory)][datetime]$EarliestWindowStart ) $rowsByKey = @{} foreach ($row in @($CoverageRows)) { if (-not $row) { continue }; $node = [string]$row.Node; $channel = [string]$row.Channel; if (-not $node -or -not $channel) { continue }; $rowsByKey[("{0}|{1}" -f $node.ToLowerInvariant(), $channel.ToLowerInvariant())] = $row } $assessmentRows = [System.Collections.Generic.List[object]]::new() foreach ($node in @($ExpectedNodes | Where-Object { $_ } | Sort-Object -Unique)) { foreach ($channel in @($ExpectedChannels | Where-Object { $_ } | Sort-Object -Unique)) { $key = "{0}|{1}" -f $node.ToLowerInvariant(), $channel.ToLowerInvariant() $row = if ($rowsByKey.ContainsKey($key)) { $rowsByKey[$key] } else { $null } $querySucceeded = [bool]($row -and $row.QuerySucceeded) $enablementKnown = [bool]($row -and $row.PSObject.Properties['IsEnabled'] -and ($row.IsEnabled -is [bool])) $isEnabled = if ($enablementKnown) { [bool]$row.IsEnabled } else { $false } $oldest = if ($row -and $row.OldestAvailable) { [datetime]$row.OldestAvailable } else { $null } $status = if (-not $row -or -not $querySucceeded) { 'Unavailable' } elseif (-not $enablementKnown) { 'Unavailable' } elseif (-not $isEnabled) { 'Disabled' } elseif (-not $oldest) { 'EnabledEmpty' } elseif ($oldest.ToUniversalTime() -gt $EarliestWindowStart.ToUniversalTime()) { 'Wrapped' } else { 'Covered' } $sufficient = ($status -in @('Covered', 'EnabledEmpty')) [void]$assessmentRows.Add([pscustomobject]@{ Node = [string]$node; Channel = [string]$channel; Status = $status; Sufficient = [bool]$sufficient; QuerySucceeded = $querySucceeded; IsEnabled = if ($enablementKnown) { $isEnabled } else { $null }; OldestAvailable = $oldest; Error = if ($row -and $row.Error) { [string]$row.Error } elseif (-not $row) { 'Coverage row was not returned.' } elseif (-not $enablementKnown) { 'Channel enablement state was not returned.' } else { '' } }) } } $rows = $assessmentRows.ToArray() $complete = ($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.Sufficient }).Count -eq 0) [pscustomobject]@{ Complete = [bool]$complete; OverallStatus = if ($complete) { 'Covered' } else { 'Incomplete' }; Rows = $rows; CoveredCount = @($rows | Where-Object Status -eq 'Covered').Count; WrappedCount = @($rows | Where-Object Status -eq 'Wrapped').Count; EnabledEmptyCount = @($rows | Where-Object Status -eq 'EnabledEmpty').Count; DisabledCount = @($rows | Where-Object Status -eq 'Disabled').Count; UnavailableCount = @($rows | Where-Object Status -eq 'Unavailable').Count } } function Get-VMCheckpointVerdictAssessment { [OutputType([pscustomobject])] param( [bool]$ConfirmingForkSignature, [bool]$HasAttachedLayers, [bool]$HasIncompleteChain, [bool]$HasStaleEvidence, [bool]$SnapshotLayerMismatch, [bool]$HasOrphans, [bool]$VssUnhealthy, [bool]$ReplicationConcern, [bool]$StorageConcern, [int]$EscalatingEventCount, [bool]$RequiredEvidenceUnavailable, [bool]$StateInconclusive ) $holdState = ($ConfirmingForkSignature -and $HasAttachedLayers) $investigate = ((-not $holdState) -and ($HasIncompleteChain -or $HasStaleEvidence -or $SnapshotLayerMismatch -or $HasOrphans -or $VssUnhealthy -or $ReplicationConcern -or $StorageConcern -or ($EscalatingEventCount -gt 0) -or $RequiredEvidenceUnavailable -or $StateInconclusive)) [pscustomobject]@{ HoldState = [bool]$holdState Investigate = [bool]$investigate Recommendation = if ($holdState) { 'HOLD STATE' } elseif ($investigate) { 'INVESTIGATE' } else { 'OK' } } } function Select-DiscoveredVMsForAudit { [OutputType([pscustomobject])] param( [object[]]$Candidates, [Nullable[int]]$Maximum ) if ($null -ne $Maximum -and ($Maximum -lt 1 -or $Maximum -gt 1000)) { throw 'Maximum must be between 1 and 1000 when specified.' } $byName = @{} foreach ($candidate in @($Candidates)) { if (-not $candidate -or -not $candidate.Name) { continue } $name = [string]$candidate.Name $key = $name.ToLowerInvariant() if (-not $byName.ContainsKey($key)) { $byName[$key] = [pscustomobject]@{ Name = $name Reasons = [System.Collections.Generic.List[string]]::new() Score = 0 } } $reason = [string]$candidate.Reason if ($reason -and -not $byName[$key].Reasons.Contains($reason)) { [void]$byName[$key].Reasons.Add($reason) } $reasonScore = if ($reason -match 'fork|3216|0x80048102') { 400 } elseif ($reason -match '19100|16300') { 300 } elseif ($reason -match '0x80070020|sharing violation') { 200 } elseif ($reason -match '19090') { 100 } else { 0 } if ($reasonScore -gt $byName[$key].Score) { $byName[$key].Score = $reasonScore } } $ranked = @($byName.Values | ForEach-Object { $orderedReasons = @($_.Reasons | Sort-Object { if ($_ -match 'fork|3216|0x80048102') { 0 } elseif ($_ -match '19100|16300') { 1 } elseif ($_ -match '0x80070020|sharing violation') { 2 } elseif ($_ -match '19090') { 3 } else { 4 } }, { $_ }) [pscustomobject]@{ Name = $_.Name Reason = if ($orderedReasons.Count -gt 0) { $orderedReasons[0] } else { 'High-risk checkpoint/merge signal' } Reasons = $orderedReasons Score = $_.Score } } | Sort-Object @{ Expression = { $_.Score }; Descending = $true }, Name) $audit = $ranked $deferred = @() if ($null -ne $Maximum) { $audit = @($ranked | Select-Object -First $Maximum) $deferred = @($ranked | Select-Object -Skip $Maximum) } [pscustomobject]@{ EligibleCount = $ranked.Count Audit = @($audit) Deferred = @($deferred) Cap = $Maximum } } function Resolve-ActiveCheckpointHistoricVerdict { [OutputType([object])] param( [bool]$HoldState, [bool]$Investigate, [bool]$LowSignalOnly, [int]$SeverityScore, [bool]$ForkConfirmed, [bool]$CoverageIncomplete ) if ($ForkConfirmed) { $HoldState = $true $Investigate = $false $LowSignalOnly = $false $SeverityScore = 100 } elseif ($CoverageIncomplete -and -not $HoldState) { $Investigate = $true $LowSignalOnly = $false if ($SeverityScore -lt 55) { $SeverityScore = 55 } } [pscustomobject]@{ HoldState = $HoldState Investigate = $Investigate LowSignalOnly = $LowSignalOnly SeverityScore = $SeverityScore } } function Complete-CheckpointHealthPassThruResult { [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory, ValueFromPipeline)] [object]$Result, [Parameter(Mandatory)] [object]$RunData ) process { $reportData = if ($Result.PSObject.Properties['ReportData']) { $Result.ReportData } else { $null } $source = if ($Result.PSObject.Properties['Source'] -and $Result.Source) { [string]$Result.Source } else { 'Input' } $recommendation = if ($Result.PSObject.Properties['Recommendation'] -and $Result.Recommendation) { [string]$Result.Recommendation } else { 'ERROR' } $detail = if ($Result.PSObject.Properties['Detail']) { [string]$Result.Detail } else { '' } if (-not $detail -and $recommendation -eq 'INVESTIGATE' -and $reportData -and $reportData.PSObject.Properties['InvestigationDrivers'] -and $reportData.InvestigationDrivers -and $reportData.InvestigationDrivers.PSObject.Properties['AssessmentText']) { $detail = [string]$reportData.InvestigationDrivers.AssessmentText } $assessmentConfidence = if ($reportData -and $reportData.PSObject.Properties['AssessmentConfidence']) { switch ([string]$reportData.AssessmentConfidence) { 'High' { 'High' } 'Moderate' { 'Moderate' } 'Complete' { 'High' } default { 'Low' } } } else { 'Low' } $nestedStatus = if ($reportData -and $reportData.PSObject.Properties['CollectionStatus']) { $reportData.CollectionStatus } else { $null } $notCollected = [pscustomobject]@{ Status = 'NotCollected' } $collectionStatus = [pscustomobject][ordered]@{ Outcome = [pscustomobject]@{ Status = $recommendation; Detail = $detail } VhdChains = if ($nestedStatus -and $nestedStatus.PSObject.Properties['VhdChains']) { $nestedStatus.VhdChains } else { $notCollected } VirtualDiskInventory = if ($nestedStatus -and $nestedStatus.PSObject.Properties['VirtualDiskInventory']) { $nestedStatus.VirtualDiskInventory } else { $notCollected } EventLogs = if ($nestedStatus -and $nestedStatus.PSObject.Properties['EventLogs']) { $nestedStatus.EventLogs } else { $notCollected } HistoricEvents = if ($nestedStatus -and $nestedStatus.PSObject.Properties['HistoricEvents']) { $nestedStatus.HistoricEvents } else { $notCollected } StateConsistency = if ($nestedStatus -and $nestedStatus.PSObject.Properties['StateConsistency']) { $nestedStatus.StateConsistency } else { $notCollected } VssWriters = if ($nestedStatus -and $nestedStatus.PSObject.Properties['VssWriters']) { $nestedStatus.VssWriters } else { $notCollected } Artifacts = if ($nestedStatus -and $nestedStatus.PSObject.Properties['Artifacts']) { $nestedStatus.Artifacts } else { $notCollected } } [pscustomobject][ordered]@{ VMName = if ($Result.PSObject.Properties['VMName']) { [string]$Result.VMName } else { '' } Cluster = if ($Result.PSObject.Properties['Cluster']) { [string]$Result.Cluster } else { '' } OwningNode = if ($Result.PSObject.Properties['OwningNode']) { [string]$Result.OwningNode } else { '' } Source = $source Recommendation = $recommendation HoldState = [bool]($Result.PSObject.Properties['HoldState'] -and $Result.HoldState) HasAttachedCheckpoints = [bool]($Result.PSObject.Properties['HasAttachedCheckpoints'] -and $Result.HasAttachedCheckpoints) HasStaleCheckpoints = [bool]($Result.PSObject.Properties['HasStaleCheckpoints'] -and $Result.HasStaleCheckpoints) HasOrphanedCheckpoints = [bool]($Result.PSObject.Properties['HasOrphanedCheckpoints'] -and $Result.HasOrphanedCheckpoints) AttachedCheckpointCount = if ($Result.PSObject.Properties['AttachedCheckpointCount']) { [int]$Result.AttachedCheckpointCount } else { 0 } StaleCheckpointCount = if ($Result.PSObject.Properties['StaleCheckpointCount']) { [int]$Result.StaleCheckpointCount } else { 0 } StaleAttachedLayerCount = if ($Result.PSObject.Properties['StaleAttachedLayerCount']) { [int]$Result.StaleAttachedLayerCount } else { 0 } SnapshotLayerMismatch = [bool]($Result.PSObject.Properties['SnapshotLayerMismatch'] -and $Result.SnapshotLayerMismatch) ConcernEventCount = if ($Result.PSObject.Properties['ConcernEventCount']) { [int]$Result.ConcernEventCount } else { 0 } AssessmentConfidence = $assessmentConfidence CollectionStatus = $collectionStatus ReportFile = if ($Result.PSObject.Properties['ReportFile']) { $Result.ReportFile } else { $null } Detail = $detail ReportData = $reportData RunData = $RunData } } } # SIG # Begin signature block # MIInKAYJKoZIhvcNAQcCoIInGTCCJxUCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCBsRrAN67Z/ETeD # plXlQqPxo9thTwSMIA1jTAM6um9t76CCDLowggX1MIID3aADAgECAhMzAAACHU0Z # yE7XD1dIAAAAAAIdMA0GCSqGSIb3DQEBCwUAMFcxCzAJBgNVBAYTAlVTMR4wHAYD # VQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBD # b2RlIFNpZ25pbmcgUENBIDIwMjQwHhcNMjYwNDE2MTg1OTQzWhcNMjcwNDE1MTg1 # OTQzWjB0MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYD # VQQDExVNaWNyb3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IB # DwAwggEKAoIBAQDQvewXxx9gZZFC6Ys1WBay8BJ8kGA4JQnH5CMafqOASlTpK9H8 # o5ZXTXt0caVQTNMUPt445wXYD+dFtaKWTwDn1I52oUSrC9vJin1Gsqt+zyKJL5Dg # 3eQXbQNR61DmMy20GLTIO3SFed9Rfi/ophgCLGFLDR3r0KvHjwMb/jYWS0celV/4 # Lz27LfAekm8v9E5IXaeiXbAUYZKK090n4CVl3JBtbN+9DtI9SNu/yjvozW52/u7R # X/Ttpa/KDlpuokZ+Zcbvmtd9ur9gFLvZzh41o9MsE/clQtdaFWGvuo6Jua/ntpgk # ey3E5/vBFe+MJPG6phdnuo6r57ZudCudiI1bAgMBAAGjggGbMIIBlzAOBgNVHQ8B # Af8EBAMCB4AwHwYDVR0lBBgwFgYKKwYBBAGCN0wIAQYIKwYBBQUHAwMwHQYDVR0O # BBYEFH6QuMwqcPG0hQlQ6c5jCtTTLrVeMEUGA1UdEQQ+MDykOjA4MR4wHAYDVQQL # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xFjAUBgNVBAUTDTIzMDAxMis1MDc1NTkw # HwYDVR0jBBgwFoAUf1k/VCHarU/vBeXmo9ctBpQSCDEwYAYDVR0fBFkwVzBVoFOg # UYZPaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jcmwvTWljcm9zb2Z0 # JTIwQ29kZSUyMFNpZ25pbmclMjBQQ0ElMjAyMDI0LmNybDBtBggrBgEFBQcBAQRh # MF8wXQYIKwYBBQUHMAKGUWh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMv # Y2VydHMvTWljcm9zb2Z0JTIwQ29kZSUyMFNpZ25pbmclMjBQQ0ElMjAyMDI0LmNy # dDAMBgNVHRMBAf8EAjAAMA0GCSqGSIb3DQEBCwUAA4ICAQBKTbYOjzwTG/DXGaz9 # s6+fQeaTtDcFmMY+5UyVFCyj7Pv+5i37qfX8lSL/tBIfYQfWsMuBQlfZurJD6r4H # VJ2CeH+1fgiq8dcHdVKoZ3Sa2qXoX3cq9iS8cVb06B7+5/XJ7I0OxHH9fDsvJ3T3 # w5V/ZtAIFmLrl+P0CtG+92uzRsn0nTbdFjOkLMLWPLAU3THohKRlSEMgFJpPkm5n # 5UAZ35xX6FWCrDLsSKb555bTifwa8mJBwdlof0bmfYidH+dxZ1FdDxvLnNl9zeKs # A4kejaaIqqIPguhwAti5Ql7BlTNoJNwxCvBmqW2MQLnCkYN/VVUsR3V2x/rcTNzo # Bf/Z/SpROvdaA2ZOOd1uioXJt3tdLQ7vHpqpib0KfWr/FWXW10q38VxfCnRQBqzb # SuztR7nEMuzX7Ck+B/XaPDXd1qh72+QYyB0Z2VzWmO9zsnb9Uq/dwu8LGeQqnyu6 # 7SDGACvnXii2fb9+US492VTnXSnFKyqwgzUyFMtZK1/sHYTv6bG4TtQUygQxTN+Z # V+aJIlKO2MqZ7bKrAnOzS9m6NgoTdWOq11bTOZwKlIEV/EhV9SWkDmdpR/hPPT2v # 6TEj4F8PT/zHjRezIU5c/DGlt/VhY/pK0XkJtEyMmmS1BMtjU/rqBZVMIm3dnxQs # /TBByr+Cf8Z1r7aifQVQ+WSqzjCCBr0wggSloAMCAQICEzMAAAA5O7Y3Gb8GHWcA # AAAAADkwDQYJKoZIhvcNAQEMBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpX # YXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQg # Q29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRl # IEF1dGhvcml0eSAyMDExMB4XDTI0MDgwODIwNTQxOFoXDTM2MDMyMjIyMTMwNFow # VzELMAkGA1UEBhMCVVMxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEo # MCYGA1UEAxMfTWljcm9zb2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAyNDCCAiIwDQYJ # KoZIhvcNAQEBBQADggIPADCCAgoCggIBANgBnB7jOMeqlRYHNa265v4IY9fH8TKh # emHfPINe1gpLaV3dhg324WwH06LcHbpnsBukCDNitryo0dtS/EW6I/yEL/bLSY8h # KpbfQuWusBPr9qazYcDxCW/qnjb5JsI1s8bNOg3bVATvQVL4tcf03aTycsz8QeCd # M0l/yHRObJ9QqazM1r6VPEOJ7LL+uEEb73w6QCuhs89a1uv1zerOYMnsneRRwCbp # yW11IcggU0cRKDDq1pjVJzIbIF6+oiXXbReOsgeI8zu1FyQfK0fVkaya8SmVHQ/t # Of23mZ4W9k0Ri22QW9p3UgSC5OUDktKxxcCmGL6tXLfOGSWHIIV4YrTJTT6PNty5 # REojHJuZHArkF9VnHTERWoTjAzfI3kP+5b4alUdhgAZ7ttOu1bVnXfHaqPYl2rPs # 20ji03LOVWsh/radgE17es5hL+t6lV0eVHrVhsssROWJuz2MXMCt7iw7lFPG9LXK # Gjsmonn2gotGdHIuEg5JnJMJVmixd5LRlkmgYRZKzhxSCwyoGIq0PhaA7Y+VPct5 # pCHkijcIIDm0nlkK+0KyepolcqGm0T/GYQRMhHJlGOOmVQop36wUVUYklUy++vDW # eEgEo4s7hxN6mIbf2MSIQ/iIfMZgJxC69oukMUXCrOC3SkE/xIkgpfl22MM1itkZ # 35nNXkMolU1lAgMBAAGjggFOMIIBSjAOBgNVHQ8BAf8EBAMCAYYwEAYJKwYBBAGC # NxUBBAMCAQAwHQYDVR0OBBYEFH9ZP1Qh2q1P7wXl5qPXLQaUEggxMBkGCSsGAQQB # gjcUAgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU # ci06AjGQQ7kUBU7h6qfHMdEjiTQwWgYDVR0fBFMwUTBPoE2gS4ZJaHR0cDovL2Ny # bC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0MjAx # MV8yMDExXzAzXzIyLmNybDBeBggrBgEFBQcBAQRSMFAwTgYIKwYBBQUHMAKGQmh0 # dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0MjAx # MV8yMDExXzAzXzIyLmNydDANBgkqhkiG9w0BAQwFAAOCAgEAFJQfOChP7onn6fLI # MKrSlN1WYKwDFgAddymOUO3FrM8d7B/W/iQ6DxXsDn7D5W4wMwYeLystcEqfkjz4 # NURRgazyMu5yRzQh4LqjA4tStTcJh1opExo7nn5PuPBYnbu0+THSuVHTe0VTTPVh # ily/piFrDo3axQ9P4C+Ol5yet+2gTfekICS5xS+cYfSIvgn0JksVBVMYVI5QFu/q # hnLhsEFEUzG8fvv0hjgkO+lkpV9ty6GkN4vdnd7ya6Q6aR9y34aiM1qmxaxBi6OU # nyNl6fkuun/diTFnYDLTppOkr/mg5WSfCiDVMNCxtj4wPKC5OmHm1DQIt/MNokbb # H3UGsFP1QbzsLocuSqLCvH09Io3fDPTmscR9Y75G4qX7RTX8AdBPo0I6OEojf39z # uFZt0qOHm65YWQE69cZM2ueE1MB05dNNgHK9gTE7zKvK/fg8B2qjW88MT/WF5V5u # vZGtqa9FSL2RazArA+rDPuf6JGYz4HpgMZHB4S6szWSKYBv0VisCzfxgeU+dquXW # 9bd0auYlOB58DPcOYKdc3Se94g+xL4pcEhbB54JOgAkwYTu/9dLeH2pDqeJZAABV # DWRQCaXfO5LgyKwKCLYXpigrZYCjUSBcr+Ve8PFWMhVTQl0v4q8J/AUmQN5W4n10 # 1cY2L4A7GTQG1h32HHAvfQESWP0xghnEMIIZwAIBATBuMFcxCzAJBgNVBAYTAlVT # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jv # c29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMjQCEzMAAAIdTRnITtcPV0gAAAAAAh0w # DQYJYIZIAWUDBAIBBQCggZAwGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwLwYJ # KoZIhvcNAQkEMSIEIErLwtWq5MCvRS9PvAfNCRT7l7HATGBK0Dd88kxQzUOEMEIG # CisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEBBQAEggEAtGq2wNgJteYphuHn # Ci4po4g5FKB+SDZRLdfPbr/dNIYzry3IqifdBHCTOyuwuWkL1jwCbrcouVac7MLo # 2efmwa9LH3Qy+Unx0viCg3llFHxNcKGI7wfekg0dZ1VNUwIpiU4kVKrrgx0yjc1q # 7iFXnGMQ44VxEJJPGsOYvTboQrQlmcX2ac4sEwTBYQ6t98M9efFqDKgEiLqG9V79 # NWaPKtxqrjICbbN+yO9xtkDk0Zv+NAx+UfEm63ZgnlZbq6IINDqUDCWC7IXF/nfL # 4KR1m0WEbGnPBNxVNHu978Z5IWGqhWutCVrDuxgffB5ZA0p25Ean0hIeqAMZcBoL # L74IEqGCF5QwgheQBgorBgEEAYI3AwMBMYIXgDCCF3wGCSqGSIb3DQEHAqCCF20w # ghdpAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFSBgsqhkiG9w0BCRABBKCCAUEEggE9 # MIIBOQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFlAwQCAQUABCDzxFRIgFB28wZA # lZDCPfGQ5cN2zJXeIQqgw2zXtnXq7QIGaoRYHFBOGBMyMDI2MDgyODE0NTUzNC40 # NTFaMASAAgH0oIHRpIHOMIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25zMScw # JQYDVQQLEx5uU2hpZWxkIFRTUyBFU046QTkzNS0wM0UwLUQ5NDcxJTAjBgNVBAMT # HE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2WgghHqMIIHIDCCBQigAwIBAgIT # MwAAAifVwIPDsS5XLQABAAACJzANBgkqhkiG9w0BAQsFADB8MQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGlt # ZS1TdGFtcCBQQ0EgMjAxMDAeFw0yNjAyMTkxOTQwMDRaFw0yNzA1MTcxOTQwMDRa # MIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQL # ExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxk # IFRTUyBFU046QTkzNS0wM0UwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1l # LVN0YW1wIFNlcnZpY2UwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDi # xWy1fDOSL4qj3A1pady+elIDLwnF3UuLzJIOWwGHcEgrxxwtnyviUIDmmxylTUl1 # u+2rBPp2zT4BwwQhvGaJpExqvPLlDFlbfmSflKI86eFqofiZ7j8NTRO4l7wGg9Nj # m+muNauTcFW2qdfIjKE950Okrm9MnMOGYy+fibNYdxTPRPq1T4MLZK3s3vdMyMEO # ldcOQkSKpxD6/1Gk6gOmCu2KgI8f0ex6vYxnKDl9W0OLSEa/6y82oIbsm+1QBifO # Q47xWKTG1CmvtGr85LzA75/MAcUmRw5/of/qET0UFV1WulMcJrI6DASAsNCNB+6W # LrotuBZAj+VMlqbn5RMZ6Q4IY7JwaAiIXh7VjxrnwUOYZG8WEGhfrA98di+7LEn9 # AqvvEOyG+UQcjVhCCbMGXigJXSApeyeWupCsD0jgQMNCxfB5BLBDWxgdY3dJBEPg # xfkgTDQLBggtVv2d5CYxHKgIItB4bI5eSb5jkIG2WotnFetT0legpw/Eozwf39ao # 6tENY21eVWIzRw/GsmvwjYQF6vVrxOD0pGVsfqGF8s3VPeY7hI2TxHFMqNA0IB/a # 2NLY7JTxYAKAP/11EJZt7xbqDLMgD1YDdGEzGpQijm3nAPCL2CebP/jmu90abJ2W # 425yglGHTI/nCBrwSpfRCgwzrfFelJaCKM6+35aFfwIDAQABo4IBSTCCAUUwHQYD # VR0OBBYEFNLW58N4MGSG6ud7jWqgT92orfReMB8GA1UdIwQYMBaAFJ+nFV0AXmJd # g/Tl0mWnG1M1GelyMF8GA1UdHwRYMFYwVKBSoFCGTmh0dHA6Ly93d3cubWljcm9z # b2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0El # MjAyMDEwKDEpLmNybDBsBggrBgEFBQcBAQRgMF4wXAYIKwYBBQUHMAKGUGh0dHA6 # Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwVGlt # ZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3J0MAwGA1UdEwEB/wQCMAAwFgYDVR0l # AQH/BAwwCgYIKwYBBQUHAwgwDgYDVR0PAQH/BAQDAgeAMA0GCSqGSIb3DQEBCwUA # A4ICAQAqncud4PSC1teb2H6nRuy7sDiKK13FXJirVB4Tfwjdo2Mb+QL4j7wZ/k4G # 9P0CANHZFrDQcK0VFDTysrYu8Z0Aha14acDZPsyIoPvAGRRhaHEuf7NckRjkfa/y # lo1KyII8jbL9N9sJAqBPL8V4FNBjljv+1GHDOw127rZz5ZSTPoAPb2SA0v5yDgcp # UMfxglPyp6cnPPoQpTtD9OGx8Dwm2P+o1TPxBIy6I0T9RauulogVCvKwflfeLTcK # AvnSG1rCjerSXmU1DNXOsAD/bsrSjgbX5mAbD7XTRMF/vawAWESFcn/BjjizxeWZ # b00aYSlkJA2rVtFlMM481aVWXdAbXPP5RzUiWTlgyHf/G7lCxHYWGIZuB13T3aI6 # Y8mEgn/ou40aiFJo8r0+i0P5GdNneWtxiR0CMKUfko+5s/73cwe1Wfp8BKXa270c # icVQasFf5sRV7pFm+V7fNRXwCu7anTOmga76zO7/2t+zOlibvphT+Q6Zd+B2qYsS # n4xBaY+YzHpnycLW5cvJyhPxBCcb1oRYfhRzCADb2utI2EtGCjc2P2ii4LyR4QMb # /n8cOweL9IqVTKKzzVk+zZJxV3vrp4LyuQXw0O30la6BcHdNAAAB9UC83zs3G9d+ # AlIfZLM97tMUNKWjbBpIirFx6LTDFXVtZQd7hqzLYByjbjH0ujCCB3EwggVZoAMC # AQICEzMAAAAVxedrngKbSZkAAAAAABUwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNV # BAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4w # HAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29m # dCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAyMDEwMB4XDTIxMDkzMDE4MjIy # NVoXDTMwMDkzMDE4MzIyNVowfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp # bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw # b3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAw # ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDk4aZM57RyIQt5osvXJHm9 # DtWC0/3unAcH0qlsTnXIyjVX9gF/bErg4r25PhdgM/9cT8dm95VTcVrifkpa/rg2 # Z4VGIwy1jRPPdzLAEBjoYH1qUoNEt6aORmsHFPPFdvWGUNzBRMhxXFExN6AKOG6N # 7dcP2CZTfDlhAnrEqv1yaa8dq6z2Nr41JmTamDu6GnszrYBbfowQHJ1S/rboYiXc # ag/PXfT+jlPP1uyFVk3v3byNpOORj7I5LFGc6XBpDco2LXCOMcg1KL3jtIckw+DJ # j361VI/c+gVVmG1oO5pGve2krnopN6zL64NF50ZuyjLVwIYwXE8s4mKyzbnijYjk # lqwBSru+cakXW2dg3viSkR4dPf0gz3N9QZpGdc3EXzTdEonW/aUgfX782Z5F37Zy # L9t9X4C626p+Nuw2TPYrbqgSUei/BQOj0XOmTTd0lBw0gg/wEPK3Rxjtp+iZfD9M # 269ewvPV2HM9Q07BMzlMjgK8QmguEOqEUUbi0b1qGFphAXPKZ6Je1yh2AuIzGHLX # pyDwwvoSCtdjbwzJNmSLW6CmgyFdXzB0kZSU2LlQ+QuJYfM2BjUYhEfb3BvR/bLU # HMVr9lxSUV0S2yW6r1AFemzFER1y7435UsSFF5PAPBXbGjfHCBUYP3irRbb1Hode # 2o+eFnJpxq57t7c+auIurQIDAQABo4IB3TCCAdkwEgYJKwYBBAGCNxUBBAUCAwEA # ATAjBgkrBgEEAYI3FQIEFgQUKqdS/mTEmr6CkTxGNSnPEP8vBO4wHQYDVR0OBBYE # FJ+nFV0AXmJdg/Tl0mWnG1M1GelyMFwGA1UdIARVMFMwUQYMKwYBBAGCN0yDfQEB # MEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMv # RG9jcy9SZXBvc2l0b3J5Lmh0bTATBgNVHSUEDDAKBggrBgEFBQcDCDAZBgkrBgEE # AYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB # /zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvXzpoYxDBWBgNVHR8ETzBNMEug # SaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9N # aWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsG # AQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jv # b0NlckF1dF8yMDEwLTA2LTIzLmNydDANBgkqhkiG9w0BAQsFAAOCAgEAnVV9/Cqt # 4SwfZwExJFvhnnJL/Klv6lwUtj5OR2R4sQaTlz0xM7U518JxNj/aZGx80HU5bbsP # MeTCj/ts0aGUGCLu6WZnOlNN3Zi6th542DYunKmCVgADsAW+iehp4LoJ7nvfam++ # Kctu2D9IdQHZGN5tggz1bSNU5HhTdSRXud2f8449xvNo32X2pFaq95W2KFUn0CS9 # QKC/GbYSEhFdPSfgQJY4rPf5KYnDvBewVIVCs/wMnosZiefwC2qBwoEZQhlSdYo2 # wh3DYXMuLGt7bj8sCXgU6ZGyqVvfSaN0DLzskYDSPeZKPmY7T7uG+jIa2Zb0j/aR # AfbOxnT99kxybxCrdTDFNLB62FD+CljdQDzHVG2dY3RILLFORy3BFARxv2T5JL5z # bcqOCb2zAVdJVGTZc9d/HltEAY5aGZFrDZ+kKNxnGSgkujhLmm77IVRrakURR6nx # t67I6IleT53S0Ex2tVdUCbFpAUR+fKFhbHP+CrvsQWY9af3LwUFJfn6Tvsv4O+S3 # Fb+0zj6lMVGEvL8CwYKiexcdFYmNcP7ntdAoGokLjzbaukz5m/8K6TT4JDVnK+AN # uOaMmdbhIurwJ0I9JZTmdHRbatGePu1+oDEzfbzL6Xu/OHBE0ZDxyKs6ijoIYn/Z # cGNTTY3ugm2lBRDBcQZqELQdVTNYs6FwZvKhggNNMIICNQIBATCB+aGB0aSBzjCB # yzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMc # TWljcm9zb2Z0IEFtZXJpY2EgT3BlcmF0aW9uczEnMCUGA1UECxMeblNoaWVsZCBU # U1MgRVNOOkE5MzUtMDNFMC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1T # dGFtcCBTZXJ2aWNloiMKAQEwBwYFKw4DAhoDFQAjHzqthPwO0GDckDMA6x54lIiM # KqCBgzCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # JjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMA0GCSqGSIb3 # DQEBCwUAAgUA7jwD2TAiGA8yMDI2MDgyODEyNTU1M1oYDzIwMjYwODI5MTI1NTUz # WjB0MDoGCisGAQQBhFkKBAExLDAqMAoCBQDuPAPZAgEAMAcCAQACAho4MAcCAQAC # AhJgMAoCBQDuPVVZAgEAMDYGCisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKg # CjAIAgEAAgMHoSChCjAIAgEAAgMBhqAwDQYJKoZIhvcNAQELBQADggEBAAWgLcZm # ZDwPvSH+FRLT+5Zd3hEDLG8eX5x/EzQuqM3dZ9/SsyCU6JbXIHWFAsZh/pJniywV # ZqXpykHyFZvEqJf8T3RaYwsvNoXXIcTfY9g0l0Mrnz6YhZhpfS21wuJCcxQ/axSY # /WVFX/0A8bpy18gtWAYrbFtIGlJeuAhW0GV/IMQdWc9/YpXxm8FuK5tqXHgWMnCP # mPW/vvQz2lIEJEdDPamF9YIgY3ES8m8NDrFaYsWa7cgR4iB6LMztVnS5BfCzxXar # lhk2VPP2mIMPbNNuS4AlDVXEY2oVajOSCDuhR+S1tZvvPqM1sa55uYqFJTrpc4az # kHIkFWKLsgwlzUwxggQNMIIECQIBATCBkzB8MQswCQYDVQQGEwJVUzETMBEGA1UE # CBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9z # b2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQ # Q0EgMjAxMAITMwAAAifVwIPDsS5XLQABAAACJzANBglghkgBZQMEAgEFAKCCAUow # GgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJBDEiBCDIt03p # uTP7rUT6jT25MxSFYtQx80n5hMl3JsfblSmD6TCB+gYLKoZIhvcNAQkQAi8xgeow # gecwgeQwgb0EIOXnARo1oVIcOLJKDqlE0adq/jZ9TXdlnXWRcXGThBFyMIGYMIGA # pH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcT # B1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UE # AxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAIn1cCDw7EuVy0A # AQAAAicwIgQgiYMqxuNfpFOpRFfwYhcktBgJkvtl8w6u2uJ47/hGf/gwDQYJKoZI # hvcNAQELBQAEggIAAceBG+iA/4KMiKL1vWNXEy7WQ6irqUF4mw206JCXrtbQAY8u # 2PYwcFWq52uvZMc1ptjArLxkhsE3bwTlqSRhuaCgMSRYe1jcTxm6BC0OlgRa17N+ # rbvT/2KUHRUjC3p3NN6+WB9Je9Zcfm4rPZ5tNz6YOIVz+EE7b9a1iLHcPVziaC2H # zBx/hQLiZEPqqcrDz9XQ96n5La6XyoO1wFftSJyhtb0frizfj6ol0vn2qphHUaxn # /iZte+DNT3t0qvdGUcelh5HxrKOkRu09iAsvWUqOOjshBq+i6O2czZi8BABERhyg # HChaxUnXdbA/M37W216oYWmy5LIWaj68Nvg7AcrMSwvznH1xlEcGfHLdTDQylbQ2 # rvw+mYpAOfQSouPK1VfOiP070mCdtx5sx4mQoug467x30K4+nRmITMdB0XgSpegU # nvi4ssxYx2D/6xzQ+QBcuDzSpxwzMTM5g8A8PS6PLa0ttpGz8CIRxxddZHJo3pvL # A6D309Ao/egDRLArxQWdDn9JbEU8oA463fXgDKyTf8x3CcbRhCd6ptFaDSqARwAU # Bzr3F0sRiAKOCkDEpGIW2UEK8yS44m1L3rxl8Qm4iVisdDMRt6kHzWrpX6CWP8I8 # l6jfVovehoFcVTglDUv7SkFVH4lmNQbr+9CLwIjm0WYjUL/FK2Y9ljO+5NM= # SIG # End signature block |