Private/AzStackHci.ConnectivityCluster.Helpers.ps1
|
# //////////////////////////////////////////////////////////////////////////// # Strict Mode v1 (PS 5.1 safe) - surfaces reads of uninitialised variables at runtime. Set-StrictMode -Version 1.0 # AzStackHci.ConnectivityCluster.Helpers.ps1 # # Cluster fan-out for Test-AzureLocalConnectivity -Scope Cluster (added v0.6.7). # # Owner constraints (see .github/copilot-instructions.md, "Owner Constraints"): # * PowerShell 5.1 target — no PS 7 features. # * NO CredSSP / WinRM auth-mode / TrustedHosts changes. # Cluster fan-out runs over standard Kerberos / Invoke-Command remoting and # assumes the operator has valid credentials when running from a node. # * NO silent shared-host security state mutation. # # Architecture: # The public cmdlet `Test-AzureLocalConnectivity -Scope Cluster` short-circuits # into `Invoke-AzureLocalConnectivityClusterFanOut` at the very top of its # `begin` block. The orchestrator: # 1. Enumerates Get-ClusterNode (UP-state nodes only). # 2. Verifies the module is installed on every node. # 3. Defaults per-node -Parallelism to 8 (each node runs its own local # Layer-7 worker sweep) unless the caller passed an explicit value. The # worker jobs run locally on each node, so the orchestrator only holds one # lightweight remoting job per node and the worker count stays bounded # per-machine — exactly like a single-node -Parallelism run. # 4. Fans out via `Invoke-Command -ComputerName $nodes` calling the same # public function with -Scope Node -PassThru -NoOutput. # 5. Collects per-node ArrayLists into a Cluster pscustomobject. # 6. Generates a tabbed HTML report (Phase C: New-ConnectivityClusterReport) # and (optionally) wires Send-DiagnosticData upload (Phase D). # # All functions in this file are PRIVATE — not exported via FunctionsToExport. # //////////////////////////////////////////////////////////////////////////// Function Invoke-AzureLocalConnectivityClusterFanOut { <# .SYNOPSIS Cluster orchestrator for Test-AzureLocalConnectivity -Scope Cluster. .DESCRIPTION Enumerates cluster nodes, fans out the connectivity test to each node via `Invoke-Command`, collects per-node ArrayList results, and produces a cluster-shaped pscustomobject for downstream reporting. Returns a `[pscustomobject]` with these properties: ClusterName : [string] name from Get-Cluster OrchestratorMachine : [string] name of the node running the orchestrator RunGuid : [guid] unique run identifier StartTime : [datetime] orchestrator start EndTime : [datetime] orchestrator end (after all nodes return) Nodes : [hashtable] {<node-name> = $perNodeResultArrayList} Errors : [array] per-node error messages (empty on success) .PARAMETER ForwardedParameters A hashtable of the parameters originally passed to Test-AzureLocalConnectivity. Cluster-mode-specific parameters (-Scope, -Parallelism, -PassThru, -NoOutput, -ExcludeUploadResults) are stripped/forced before fanning out. .PARAMETER ExportPath The top-level path to write the cluster report HTML/JSON to. If not provided, defaults to `$env:USERPROFILE\AzStackHci.DiagnosticSettings\<ClusterName>\<timestamp>`. .PARAMETER NoOutput When set, suppresses orchestrator console output (per-node calls already use -NoOutput). .PARAMETER ExcludeUploadResults When set, skips the Send-DiagnosticData upload step at the end. .PARAMETER InstallMissingModuleOnNodes When set, side-loads the orchestrator's exact module version onto any node that is missing the module or has a different version (drift), copying over a PSSession from the orchestrator's installed module folder. Opt-in — never mutates nodes silently. Without it, missing/drifted nodes cause a graceful pre-flight failure. #> [CmdletBinding()] [OutputType([pscustomobject])] param( [Parameter(Mandatory)] [hashtable]$ForwardedParameters, [Parameter(Mandatory=$false)] [string]$ExportPath, [Parameter(Mandatory=$false)] [switch]$NoOutput, [Parameter(Mandatory=$false)] [switch]$ExcludeUploadResults, [Parameter(Mandatory=$false)] [switch]$InstallMissingModuleOnNodes ) # ── Verify cluster cmdlets are available ───────────────────────────────── # NOTE: Test-CommandExists declares Param ($command) — it has NO -CommandName # parameter. Calling with -CommandName silently leaves $command empty (no # CmdletBinding strict-binding), Get-Command '' returns nothing, and the # precheck wrongly throws even when the cmdlet exists. Use positional binding. if (-not (Test-CommandExists 'Get-Cluster')) { Throw "Get-Cluster cmdlet not found. The FailoverClusters PowerShell module is required for -Scope Cluster. Install RSAT-Clustering or run from a cluster node." } if (-not (Test-CommandExists 'Get-ClusterNode')) { Throw "Get-ClusterNode cmdlet not found. The FailoverClusters PowerShell module is required for -Scope Cluster." } # ── Enumerate the cluster ──────────────────────────────────────────────── [string]$clusterName = '' [array]$nodes = @() try { $clusterObj = Get-Cluster -ErrorAction Stop $clusterName = $clusterObj.Name # Only fan out to UP nodes — DOWN/PAUSED nodes would just produce error rows. $nodes = @(Get-ClusterNode -ErrorAction Stop | Where-Object { $_.State -eq 'Up' } | Select-Object -ExpandProperty Name) } catch { Throw "Cluster enumeration failed: $($_.Exception.Message). -Scope Cluster requires the current host to be a member of an active failover cluster (or to have RSAT remote access to one)." } if ($nodes.Count -eq 0) { Throw "Get-ClusterNode returned 0 UP nodes for cluster '$clusterName'. Cannot fan out." } if (-not $NoOutput.IsPresent) { Write-HostAzS "════════════════════════════════════════════════════════════════════" Write-HostAzS " Cluster connectivity test — Scope: Cluster" Write-HostAzS " Cluster: $clusterName Orchestrator: $env:COMPUTERNAME" Write-HostAzS " Nodes ($($nodes.Count)): $($nodes -join ', ')" Write-HostAzS "════════════════════════════════════════════════════════════════════" } # ── Verify the module is installed on every node ───────────────────────── # Categorise failures so the user gets actionable guidance: # * Remoting failures (WinRM access denied, name resolution, firewall) # usually mean the orchestrator isn't an admin on the remote node — NOT # a missing module. Surface those separately to avoid the confusing # "install the module" prompt when the real fix is admin / network. # * Truly missing modules need a different remediation (Install-Module). # # On any blocking failure (missing module and/or remoting failure) we DO NOT # Throw a raw exception — that surfaces a confusing red script stack trace to # the operator. Instead we print a clean, formatted summary that names the # exact node(s) at fault with remediation guidance, also emit a single # Write-Error so scripted / -NoOutput callers can still detect the failure, # then return $null to stop the cluster fan-out gracefully. if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "Verifying 'AzStackHci.DiagnosticSettings' module (exact version) is installed on each cluster node..." -ForegroundColor Cyan } $orchestratorVersion = Get-LoadedModuleVersion -Name 'AzStackHci.DiagnosticSettings' [string]$orchestratorVersionString = if ($orchestratorVersion) { $orchestratorVersion.ToString() } else { $null } # Probe every node for the FULL list of installed versions so we can enforce an # EXACT version match against the orchestrator. This prevents a silent mixed-version # cluster report (different nodes running different module code, merged into one # report). Classification per node: # OK — the orchestrator's exact version is present # Missing — no version of the module at all # Drifted — has the module, but NOT the orchestrator's exact version # Remoting failures (WinRM / admin / firewall) are tracked separately — side-loading # cannot fix those, so the remediation hint must not point at -InstallMissingModuleOnNodes. $okNodes = @() $missingNodes = @() $driftedNodes = @() # display strings: "node (has X, Y; orchestrator has Z)" $driftedNodeNames = @() # bare names for side-load targeting $remotingFailures = @() foreach ($node in $nodes) { try { $remoteVersions = @(Invoke-Command -ComputerName $node -ScriptBlock { Get-Module -ListAvailable -Name 'AzStackHci.DiagnosticSettings' -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Version | ForEach-Object { $_.ToString() } } -ErrorAction Stop) $remoteVersions = @($remoteVersions | Where-Object { $_ }) # drop any $null/empty if ($remoteVersions.Count -eq 0) { $missingNodes += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Module NOT installed" -ForegroundColor Red } } elseif (-not $orchestratorVersionString) { # Orchestrator version is indeterminate — fall back to "any version present is OK" # rather than falsely reporting drift on every node. $okNodes += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Module present (orchestrator version unknown — exact-match check skipped)" -ForegroundColor Yellow } } elseif ($remoteVersions -contains $orchestratorVersionString) { $okNodes += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Module v$orchestratorVersionString present" -ForegroundColor Green } } else { $driftedNodes += "$node (has $($remoteVersions -join ', '); orchestrator has $orchestratorVersionString)" $driftedNodeNames += $node if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Version drift — has $($remoteVersions -join ', '), needs $orchestratorVersionString" -ForegroundColor Yellow } } } catch { $remotingFailures += "$node ($($_.Exception.Message))" if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] Remoting failed: $($_.Exception.Message)" -ForegroundColor Red } } } # ── Optional remediation: side-load the orchestrator's exact version ───── # When -InstallMissingModuleOnNodes is specified, copy the orchestrator's installed # module folder (exact version) to every Missing / Drifted node over a PSSession, # then reclassify successes as OK. This is a deliberate, OPT-IN host mutation (see # Owner Constraints) — never performed silently. We side-load rather than # Install-Module from PSGallery because Azure Local nodes are frequently air-gapped, # and copying guarantees a byte-identical version match with the orchestrator (no # PSGallery drift, no NuGet provider, no internet dependency). Remoting-failed nodes # are NOT targeted — the PSSession would fail for the same reason the probe did. if ($InstallMissingModuleOnNodes.IsPresent -and ($missingNodes.Count -gt 0 -or $driftedNodeNames.Count -gt 0)) { if (-not $orchestratorVersionString) { if (-not $NoOutput.IsPresent) { Write-HostAzS " -InstallMissingModuleOnNodes specified but the orchestrator module version is unknown — cannot side-load. Skipping remediation." -ForegroundColor Yellow } } else { # Resolve the orchestrator's on-disk module folder for the exact version. # Prefer the -ListAvailable entry matching the version; fall back to the # loaded module's ModuleBase (covers dev / non-PSModulePath installs). $sourceModuleBase = (Get-Module -Name 'AzStackHci.DiagnosticSettings' -ListAvailable -ErrorAction SilentlyContinue | Where-Object { $_.Version.ToString() -eq $orchestratorVersionString } | Sort-Object Version -Descending | Select-Object -First 1).ModuleBase if (-not $sourceModuleBase -or -not (Test-Path -Path $sourceModuleBase)) { $sourceModuleBase = (Get-Module -Name 'AzStackHci.DiagnosticSettings' | Sort-Object Version -Descending | Select-Object -First 1).ModuleBase } if (-not $sourceModuleBase -or -not (Test-Path -Path $sourceModuleBase)) { if (-not $NoOutput.IsPresent) { Write-HostAzS " -InstallMissingModuleOnNodes specified but the orchestrator module folder could not be located on disk — cannot side-load. Skipping remediation." -ForegroundColor Yellow } } else { $targets = @(@($missingNodes + $driftedNodeNames) | Select-Object -Unique) if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS " -InstallMissingModuleOnNodes: side-loading AzStackHci.DiagnosticSettings v$orchestratorVersionString to $($targets.Count) node(s) from:" -ForegroundColor Cyan Write-HostAzS " $sourceModuleBase" -ForegroundColor DarkGray } foreach ($targetNode in $targets) { if (-not $NoOutput.IsPresent) { Write-HostAzS " [$targetNode] Copying module v$orchestratorVersionString..." -ForegroundColor Cyan } $copied = Copy-AzStackHciModuleToNode -NodeName $targetNode -SourceModuleBase $sourceModuleBase -Version $orchestratorVersion if ($copied) { if (-not $NoOutput.IsPresent) { Write-HostAzS " [$targetNode] Side-load OK — module v$orchestratorVersionString now present" -ForegroundColor Green } $okNodes += $targetNode $missingNodes = @($missingNodes | Where-Object { $_ -ne $targetNode }) $driftedNodeNames = @($driftedNodeNames | Where-Object { $_ -ne $targetNode }) $driftedNodes = @($driftedNodes | Where-Object { $_ -notlike "$targetNode (*" }) } else { if (-not $NoOutput.IsPresent) { Write-HostAzS " [$targetNode] Side-load FAILED — re-run with -Verbose for the copy error" -ForegroundColor Red } } } } } } # ── Graceful failure when one or more nodes cannot run the test ────────── if ($missingNodes.Count -gt 0 -or $driftedNodes.Count -gt 0 -or $remotingFailures.Count -gt 0) { $moduleProblem = ($missingNodes.Count -gt 0 -or $driftedNodes.Count -gt 0) if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "════════════════════════════════════════════════════════════════════" -ForegroundColor Red Write-HostAzS " Cluster connectivity test cannot continue — pre-flight checks failed" -ForegroundColor Red Write-HostAzS "════════════════════════════════════════════════════════════════════" -ForegroundColor Red if ($missingNodes.Count -gt 0) { Write-HostAzS "" Write-HostAzS " The 'AzStackHci.DiagnosticSettings' module is NOT installed on $($missingNodes.Count) node(s):" -ForegroundColor Red foreach ($mn in $missingNodes) { Write-HostAzS " - $mn" -ForegroundColor Yellow } } if ($driftedNodes.Count -gt 0) { Write-HostAzS "" Write-HostAzS " The module version does NOT match the orchestrator (v$orchestratorVersionString) on $($driftedNodes.Count) node(s):" -ForegroundColor Red foreach ($dn in $driftedNodes) { Write-HostAzS " - $dn" -ForegroundColor Yellow } Write-HostAzS "" Write-HostAzS " All nodes must run the SAME module version, otherwise the cluster report can" -ForegroundColor Yellow Write-HostAzS " silently blend results collected by different module versions." -ForegroundColor Yellow } if ($moduleProblem) { Write-HostAzS "" if ($InstallMissingModuleOnNodes.IsPresent) { Write-HostAzS " -InstallMissingModuleOnNodes was specified but the side-load did not succeed on the" -ForegroundColor Yellow Write-HostAzS " node(s) above. Re-run with -Verbose for the copy error, or install the exact version" -ForegroundColor Yellow Write-HostAzS " manually on each node:" -ForegroundColor Yellow Write-HostAzS " Install-Module -Name AzStackHci.DiagnosticSettings -RequiredVersion $orchestratorVersionString -Scope AllUsers" -ForegroundColor Yellow } else { Write-HostAzS " TIP: re-run with -InstallMissingModuleOnNodes to automatically copy the orchestrator's" -ForegroundColor Green Write-HostAzS " exact version (v$orchestratorVersionString) to the affected node(s), e.g.:" -ForegroundColor Green Write-HostAzS " Test-AzureLocalConnectivity -Scope Cluster -InstallMissingModuleOnNodes ..." -ForegroundColor Green Write-HostAzS "" Write-HostAzS " Or install the exact version manually on each node:" -ForegroundColor Yellow Write-HostAzS " Install-Module -Name AzStackHci.DiagnosticSettings -RequiredVersion $orchestratorVersionString -Scope AllUsers" -ForegroundColor Yellow } } if ($remotingFailures.Count -gt 0) { Write-HostAzS "" Write-HostAzS " PowerShell remoting (Invoke-Command) failed on $($remotingFailures.Count) node(s):" -ForegroundColor Red foreach ($rf in $remotingFailures) { Write-HostAzS " - $rf" -ForegroundColor Yellow } Write-HostAzS "" Write-HostAzS " -Scope Cluster requires the orchestrator session to be Administrator on every" -ForegroundColor Yellow Write-HostAzS " cluster node, with WinRM reachable (default port 5985 TCP). Verify with:" -ForegroundColor Yellow Write-HostAzS " Test-WSMan -ComputerName <nodeName>" -ForegroundColor Yellow } Write-HostAzS "" } # Single error-stream message so scripted / -NoOutput callers detect the failure. $errorParts = @() if ($missingNodes.Count -gt 0) { $errorParts += "module not installed on: $($missingNodes -join ', ')" } if ($driftedNodes.Count -gt 0) { $errorParts += "module version drift on: $($driftedNodeNames -join ', ')" } if ($remotingFailures.Count -gt 0) { $errorParts += "remoting failed on: $($remotingFailures -join '; ')" } $hint = if ($moduleProblem -and -not $InstallMissingModuleOnNodes.IsPresent) { " Re-run with -InstallMissingModuleOnNodes to copy the orchestrator's exact version (v$orchestratorVersionString) to the affected node(s)." } else { '' } Write-Error "Test-AzureLocalConnectivity -Scope Cluster aborted — $($errorParts -join ' | ').$hint" return $null } # ── Build the per-node parameter set (strip cluster-specific switches) ── # NOTE: 'Parallelism' is intentionally NOT stripped — each node runs its own # Layer-7 sweep in parallel. The worker Start-Jobs run LOCALLY on each node # (children of that node's wsmprovhost), so the orchestrator only holds one # lightweight remoting job per node; the worker count is bounded per-machine # exactly like a normal single-node -Parallelism run. Version parity (enforced # by the pre-flight) guarantees every node's workers import identical code. $perNodeParams = @{} foreach ($k in $ForwardedParameters.Keys) { if ($k -notin @('Scope','PassThru','NoOutput','ExcludeUploadResults','InstallMissingModuleOnNodes')) { $perNodeParams[$k] = $ForwardedParameters[$k] } } $perNodeParams['Scope'] = 'Node' if (-not $perNodeParams.ContainsKey('Parallelism')) { # Cluster default: 8 local workers per node (only applied when the caller # did NOT pass an explicit -Parallelism). A single-node run still defaults # to 1; the higher default here just speeds up the per-node sweep. $perNodeParams['Parallelism'] = 8 } $perNodeParams['PassThru'] = $true $perNodeParams['NoOutput'] = $true $perNodeParams['ExcludeUploadResults'] = $true # orchestrator handles upload (Phase D) # ── Resolve the export path on the orchestrator ────────────────────────── # Default to the SAME ProgramData location single-node runs use, so cluster # reports land alongside single-node reports (C:\ProgramData\...) instead of # under the operator's roaming profile (C:\Users\<admin>\...). if (-not $ExportPath) { $ExportPath = 'C:\ProgramData\AzStackHci.DiagnosticSettings' } if (-not (Test-Path -Path $ExportPath)) { New-Item -Path $ExportPath -ItemType Directory -Force | Out-Null } # ── Fan out: run every node IN PARALLEL via Start-Job ─────────────────── # Each node gets its own background job (process-isolated, PS 5.1 safe — no # ForEach-Object -Parallel). The job wraps an Invoke-Command -ComputerName so # the actual test runs on the node; Start-Job isolation keeps each node's # remote Write-Progress / host output out of the orchestrator console, and # lets us show a single cluster-level progress bar instead of per-endpoint # noise. All nodes launch at once (unbounded) — each node runs its own local # Layer-7 worker sweep (per-node -Parallelism, default 8), so the worker # count stays bounded per-machine. $nodeJobScript = { param($NodeName, $Params, $RequiredVersion) $jobStart = Get-Date try { $remote = Invoke-Command -ComputerName $NodeName -ScriptBlock { param($P, $Ver) # Pin the import to the orchestrator's EXACT version so every node runs # byte-identical code even if a node also has other versions installed # (the pre-flight has already guaranteed this version is present here). if ($Ver) { Import-Module 'AzStackHci.DiagnosticSettings' -RequiredVersion $Ver -Force } else { Import-Module 'AzStackHci.DiagnosticSettings' -Force } $raw = Test-AzureLocalConnectivity @P # Re-emit the per-row results PLUS the run-level telemetry that lives as # NoteProperties on the returned ArrayList. Invoke-Command ENUMERATES a # returned collection on the way back, which STRIPS container NoteProperties # (DownloadSpeed, Layer7WallClockSeconds, ...). Capturing them here as scalar # properties on a pscustomobject preserves them across the remoting CliXml # boundary. [pscustomobject]@{ Rows = $raw DownloadSpeed = $raw.DownloadSpeed RequestMethod = $raw.RequestMethod Parallelism = $raw.Parallelism TotalDurationSeconds = $raw.TotalDurationSeconds Layer7TotalDurationSeconds = $raw.Layer7TotalDurationSeconds Layer7WallClockSeconds = $raw.Layer7WallClockSeconds Layer7TestedEndpoints = $raw.Layer7TestedEndpoints } } -ArgumentList $Params, $RequiredVersion -ErrorAction Stop # Return a FLAT wrapper (Rows one level deep, same nesting as the working # single-hop design) so the extra Start-Job CliXml boundary preserves the # already-deserialised flat row property bags. [pscustomobject]@{ Node = $NodeName Error = $null Duration = ((Get-Date) - $jobStart).ToString() Rows = $remote.Rows DownloadSpeed = $remote.DownloadSpeed RequestMethod = $remote.RequestMethod Parallelism = $remote.Parallelism TotalDurationSeconds = $remote.TotalDurationSeconds Layer7TotalDurationSeconds = $remote.Layer7TotalDurationSeconds Layer7WallClockSeconds = $remote.Layer7WallClockSeconds Layer7TestedEndpoints = $remote.Layer7TestedEndpoints } } catch { [pscustomobject]@{ Node = $NodeName Error = $_.Exception.Message Duration = ((Get-Date) - $jobStart).ToString() Rows = $null } } } $clusterStart = Get-Date $perNodeResults = @{} $perNodeErrors = @{} $perNodeDurations = @{} $perNodeDownloadSpeed = @{} # per-node download-speed result (string, e.g. '123.4 Mbps') $perNodeTelemetry = @{} # per-node run-level telemetry (timing + RequestMethod) # Launch all node jobs at once. $jobMap = @{} foreach ($node in $nodes) { $jobMap[$node] = Start-Job -Name "AzSHciConn_$node" -ScriptBlock $nodeJobScript ` -ArgumentList $node, $perNodeParams, $orchestratorVersionString } if (-not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "Fanned out to $($nodes.Count) node(s) in parallel: $($nodes -join ', ')" -ForegroundColor Cyan } # ── Poll for completion + show a node-level progress bar ───────────────── $jobList = @($jobMap.Values) $totalNodes = $nodes.Count do { $pending = @($jobList | Where-Object { $_.State -eq 'Running' -or $_.State -eq 'NotStarted' }) $doneCount = $totalNodes - $pending.Count if (-not $NoOutput.IsPresent) { $pendingNodes = @($nodes | Where-Object { $jobMap[$_].State -eq 'Running' -or $jobMap[$_].State -eq 'NotStarted' }) $pct = if ($totalNodes -gt 0) { [int](($doneCount / $totalNodes) * 100) } else { 100 } $statusText = if ($pendingNodes.Count -gt 0) { "Waiting on $($pendingNodes.Count) of $totalNodes node(s): $($pendingNodes -join ', ')" } else { 'Finalising...' } Write-Progress -Activity "Cluster connectivity test ($doneCount of $totalNodes nodes complete)" -Status $statusText -PercentComplete $pct } if ($pending.Count -gt 0) { $null = Wait-Job -Job $jobList -Any -Timeout 2 } } while (@($jobList | Where-Object { $_.State -eq 'Running' -or $_.State -eq 'NotStarted' }).Count -gt 0) if (-not $NoOutput.IsPresent) { Write-Progress -Activity "Cluster connectivity test" -Completed } # ── Receive each job + attribute results back to its node ──────────────── foreach ($node in $nodes) { $job = $jobMap[$node] $jobOutput = $null try { $jobOutput = Receive-Job -Job $job -ErrorAction Stop } catch { $jobOutput = $null } Remove-Job -Job $job -Force -ErrorAction SilentlyContinue # Start-Job can emit multiple objects; pick our wrapper (it carries a 'Node' prop). $nodeWrapper = $null if ($jobOutput) { $nodeWrapper = @($jobOutput | Where-Object { $_ -and $_.PSObject.Properties['Node'] }) | Select-Object -First 1 } if ($nodeWrapper -and (-not $nodeWrapper.Error) -and $nodeWrapper.PSObject.Properties['Rows'] -and $nodeWrapper.Rows) { $perNodeResults[$node] = @($nodeWrapper.Rows) $perNodeDownloadSpeed[$node] = $nodeWrapper.DownloadSpeed $perNodeTelemetry[$node] = [pscustomobject]@{ DownloadSpeed = $nodeWrapper.DownloadSpeed RequestMethod = $nodeWrapper.RequestMethod Parallelism = $nodeWrapper.Parallelism TotalDurationSeconds = $nodeWrapper.TotalDurationSeconds Layer7TotalDurationSeconds = $nodeWrapper.Layer7TotalDurationSeconds Layer7WallClockSeconds = $nodeWrapper.Layer7WallClockSeconds Layer7TestedEndpoints = $nodeWrapper.Layer7TestedEndpoints } $perNodeErrors[$node] = @() if (-not $NoOutput.IsPresent) { $rowCount = @($perNodeResults[$node]).Count $dlNote = if ($perNodeDownloadSpeed[$node]) { " (download speed: $($perNodeDownloadSpeed[$node]))" } else { '' } Write-HostAzS " [$node] Returned $rowCount result row(s).$dlNote" -ForegroundColor Green } } else { $perNodeResults[$node] = $null $perNodeDownloadSpeed[$node] = $null $perNodeTelemetry[$node] = $null $errMsg = if ($nodeWrapper -and $nodeWrapper.Error) { $nodeWrapper.Error } else { "No result returned from node '$node' (job produced no output)." } $perNodeErrors[$node] = @($errMsg) if (-not $NoOutput.IsPresent) { Write-HostAzS " [$node] FAILED: $errMsg" -ForegroundColor Red } } if ($nodeWrapper -and $nodeWrapper.PSObject.Properties['Duration'] -and $nodeWrapper.Duration) { $perNodeDurations[$node] = try { [TimeSpan]::Parse($nodeWrapper.Duration) } catch { [TimeSpan]::Zero } } else { $perNodeDurations[$node] = [TimeSpan]::Zero } } $clusterEnd = Get-Date # ── Console summary: per-node connectivity FAILURES only ───────────────── # -Scope Node prints every endpoint result inline; -Scope Cluster suppresses the # per-node console (Start-Job isolation keeps each node's endpoint sweep out of the # orchestrator). To restore the at-a-glance failure visibility of a Node-scope run, # surface ONLY the failing endpoints here, attributed to the node they came from. # (The full per-endpoint detail still lives in the tabbed HTML / JSON report.) if (-not $NoOutput.IsPresent) { $failureLines = [System.Collections.Generic.List[string]]::new() foreach ($node in $nodes) { $rows = @($perNodeResults[$node]) if ($rows.Count -eq 0) { continue } $nodeFailures = @($rows | Where-Object { $_ -and ( ($_.PSObject.Properties['TCPStatus'] -and $_.TCPStatus -eq 'Failed') -or ($_.PSObject.Properties['Layer7Status'] -and $_.Layer7Status -eq 'Failed') ) }) foreach ($f in $nodeFailures) { $u = if ($f.PSObject.Properties['URL']) { $f.URL } else { '<unknown>' } $p = if ($f.PSObject.Properties['Port']) { $f.Port } else { '' } $tcp = if ($f.PSObject.Properties['TCPStatus']) { $f.TCPStatus } else { '' } $l7 = if ($f.PSObject.Properties['Layer7Status']) { $f.Layer7Status } else { '' } # NOTE: wrap the -f in EXTRA parens — commas inside method () are arg # separators, so '... -f a, b' would pass b as a 2nd .Add() argument. # Leading tab indents the row under the banner so it stands out in console. $failureLines.Add(("`t [{0}] {1}:{2} TCP={3} L7={4}" -f $node, $u, $p, $tcp, $l7)) } } # Header banner (matches the module's //// section style) plus a tab indent so the # cluster failure summary is easy to spot in a busy console — mirroring the # at-a-glance "Test results summary:" block that -Scope Node prints. Write-HostAzS "`n`t//////////////// Connectivity Test Results ////////////////`n" -ForegroundColor Cyan if ($failureLines.Count -gt 0) { Write-HostAzS "`tConnectivity FAILURES ($($failureLines.Count) across $($nodes.Count) node(s)):" -ForegroundColor Red foreach ($line in $failureLines) { Write-HostAzS $line -ForegroundColor Red } } else { Write-HostAzS "`tNo connectivity failures detected on any node." -ForegroundColor Green } Write-HostAzS "" } # ── Build cluster-shaped result object ─────────────────────────────────── $clusterResult = [pscustomobject]@{ ClusterName = $clusterName OrchestratorMachine = $env:COMPUTERNAME RunGuid = [guid]::NewGuid() StartTime = $clusterStart EndTime = $clusterEnd Duration = ($clusterEnd - $clusterStart) Nodes = $perNodeResults NodeDurations = $perNodeDurations NodeDownloadSpeeds = $perNodeDownloadSpeed NodeTelemetry = $perNodeTelemetry Errors = $perNodeErrors ExportPath = $ExportPath } # ── Phase C — Generate cluster report (tabbed HTML, or merged CSV) ─────── # Honour the caller's -OutputFormat (HTML default; CSV writes a merged CSV). # JSON is always emitted alongside. AzureRegion isn't needed for cluster file # naming (cluster name only), so we don't depend on it here. $clusterOutputFormat = if ($ForwardedParameters.ContainsKey('OutputFormat') -and $ForwardedParameters['OutputFormat']) { [string]$ForwardedParameters['OutputFormat'] } else { 'HTML' } try { $reportPath = New-ConnectivityClusterReport -ClusterResult $clusterResult -ExportPath $ExportPath -OutputFormat $clusterOutputFormat if ($reportPath -and -not $NoOutput.IsPresent) { Write-HostAzS "" Write-HostAzS "Cluster connectivity report: $reportPath" -ForegroundColor Cyan } } catch { Write-Warning "New-ConnectivityClusterReport failed: $($_.Exception.Message)" } # ── Phase D — Upload bundle (Send-DiagnosticData integration) ─────────── # Test-CommandExists takes the command positionally (Param ($command)); -CommandName # is NOT a valid parameter and would silently bind to nothing. if (-not $ExcludeUploadResults.IsPresent -and (Test-CommandExists 'Send-DiagnosticData')) { try { Invoke-UploadDiagnosticResults ` -FolderToUpload $ExportPath ` -MessageToDisplay "Send the cluster connectivity results bundle to Microsoft? [Y/N]" ` -Context 'Cluster Connectivity Test results' ` -ErrorAction Stop } catch { Write-Warning "Cluster upload via Send-DiagnosticData failed: $($_.Exception.Message). Bundle remains at: $ExportPath" } } return $clusterResult } Function Copy-AzStackHciModuleToNode { <# .SYNOPSIS Side-loads (copies) the orchestrator's installed AzStackHci.DiagnosticSettings module folder onto a remote cluster node over a PSSession. .DESCRIPTION Used by Invoke-AzureLocalConnectivityClusterFanOut when -InstallMissingModuleOnNodes is specified. Copies the orchestrator's exact versioned module folder (`...\Modules\AzStackHci.DiagnosticSettings\<version>`) to the same AllUsers module path on the target node, guaranteeing a byte-identical version match without any PowerShell Gallery / internet dependency (Azure Local nodes are frequently air-gapped). The copy is performed over a `New-PSSession` (NOT the `C$` admin share) so it depends only on the same WinRM remoting already required by -Scope Cluster — no extra SMB / firewall surface. Any existing copy of the SAME version on the node is removed first to avoid a partial-merge. Copied files are Unblock-File'd, then the version is re-verified via `Get-Module -ListAvailable` on the node. Owner constraints: this is an OPT-IN host mutation only; it does not change CredSSP, WinRM auth mode, TrustedHosts, or any shared security state. Returns $true if the exact version is present on the node after the copy, else $false. .PARAMETER NodeName The remote node to copy the module to. .PARAMETER SourceModuleBase The orchestrator's on-disk versioned module folder, e.g. `C:\Program Files\WindowsPowerShell\Modules\AzStackHci.DiagnosticSettings\0.6.7`. .PARAMETER Version The exact module version being side-loaded (used for the destination subfolder and post-copy verification). .NOTES SECURITY / TRUST BOUNDARY: this copies the orchestrator's local module folder ($SourceModuleBase) to every node's %ProgramFiles%\WindowsPowerShell\Modules and imports it. Whatever is in the orchestrator's module install is what gets pushed to all nodes — run the side-load (-InstallMissingModuleOnNodes) only from a trusted, clean module install. Uses default Kerberos remoting (no CredSSP) and fails closed (returns $false) on any error. #> [CmdletBinding()] [OutputType([bool])] param( [Parameter(Mandatory)] [string]$NodeName, [Parameter(Mandatory)] [string]$SourceModuleBase, [Parameter(Mandatory)] [version]$Version ) $session = $null try { $session = New-PSSession -ComputerName $NodeName -ErrorAction Stop # Destination module-name root on the node (PS 5.1 AllUsers path). Resolved on the # node itself so a non-default %ProgramFiles% is honoured. $destModuleNameRoot = Invoke-Command -Session $session -ScriptBlock { Join-Path $env:ProgramFiles 'WindowsPowerShell\Modules\AzStackHci.DiagnosticSettings' } $verString = $Version.ToString() # Ensure the module-name root exists and clear any partial copy of this exact version. Invoke-Command -Session $session -ScriptBlock { param($root, $ver) if (-not (Test-Path -Path $root)) { New-Item -Path $root -ItemType Directory -Force | Out-Null } $verPath = Join-Path $root $ver # Defence-in-depth: only delete when the leaf is a well-formed version string # (e.g. 0.6.7 / 0.6.7.0) AND the resolved path is still under the module-name # root, so a malformed $ver can never escalate this into deleting an # unintended path on the remote node. if (($ver -match '^\d+\.\d+\.\d+(\.\d+)?$') -and ($verPath -like (Join-Path $root '*')) -and (Test-Path -Path $verPath)) { Remove-Item -Path $verPath -Recurse -Force -ErrorAction SilentlyContinue } } -ArgumentList $destModuleNameRoot, $verString -ErrorAction Stop # Copy the versioned folder INTO the module-name root → ...\<ModuleName>\<version>. Copy-Item -Path $SourceModuleBase -Destination $destModuleNameRoot -ToSession $session -Recurse -Force -ErrorAction Stop # Unblock copied files and confirm the exact version is now discoverable on the node. $ok = Invoke-Command -Session $session -ScriptBlock { param($root, $ver) $verPath = Join-Path $root $ver Get-ChildItem -Path $verPath -Recurse -ErrorAction SilentlyContinue | Unblock-File -ErrorAction SilentlyContinue $found = Get-Module -ListAvailable -Name 'AzStackHci.DiagnosticSettings' -ErrorAction SilentlyContinue | Where-Object { $_.Version.ToString() -eq $ver } [bool]$found } -ArgumentList $destModuleNameRoot, $verString -ErrorAction Stop return [bool]$ok } catch { Write-Verbose "Copy-AzStackHciModuleToNode: side-load to '$NodeName' failed: $($_.Exception.Message)" return $false } finally { if ($session) { Remove-PSSession -Session $session -ErrorAction SilentlyContinue } } } Function New-ConnectivityClusterReport { <# .SYNOPSIS Builds the tabbed HTML cluster report from a Cluster-shaped PassThru object. .DESCRIPTION Mirrors the structure of `New-OsConfigReport` (Private/AzStackHci.OsConfigReport.Helpers.ps1) but specialised for connectivity-test data. Sections: * Status Overview (top of Summary tab) — Node | Tested | Passed | Failed | SSL-Inspected | Private-Link | Status badge. * Per-node tabs — same tabular result layout the existing CSV/HTML emits today, scoped to that node's PassThru ArrayList. Returns the full HTML report file path on success, or $null on failure. .PARAMETER ClusterResult The pscustomobject produced by Invoke-AzureLocalConnectivityClusterFanOut. .PARAMETER ExportPath The directory to write the HTML/JSON files to. #> [CmdletBinding()] [OutputType([string])] param( [Parameter(Mandatory)] [pscustomobject]$ClusterResult, [Parameter(Mandatory)] [string]$ExportPath, [Parameter(Mandatory=$false)] [ValidateSet('HTML','CSV')] [string]$OutputFormat = 'HTML' ) if (-not (Test-Path -Path $ExportPath)) { New-Item -Path $ExportPath -ItemType Directory -Force | Out-Null } # Nested HTML-encode helper. The ConvertTo-HtmlSafe in OsConfigReport.Helpers.ps1 # is nested inside New-OsConfigReport, so it isn't visible from this scope; we # define our own with the same name + signature so all the call sites below read # cleanly. PowerShell function lookup resolves nested functions before sibling # function names, so this does NOT shadow the OsConfigReport version. function ConvertTo-HtmlSafe { param([string]$Text) if (-not $Text) { return '' } return [System.Net.WebUtility]::HtmlEncode($Text) } # File naming: cluster name only (no orchestrator host name), matching the # single-node AzureLocal_ConnectivityTest_* convention so all reports group # together in C:\ProgramData\AzStackHci.DiagnosticSettings. $reportStamp = Get-Date -Format 'yyyy-MM-dd-HH-mm-ss' $safeClusterName = ([string]$ClusterResult.ClusterName) -replace '[^A-Za-z0-9._-]', '_' $baseName = "AzureLocal_ConnectivityTest_Cluster_${safeClusterName}_${reportStamp}" $reportExt = if ($OutputFormat -eq 'CSV') { '.csv' } else { '.html' } $reportFile = Join-Path $ExportPath ($baseName + $reportExt) $jsonFile = Join-Path $ExportPath ($baseName + '.json') $sb = [System.Text.StringBuilder]::new() # ── HTML head + CSS ────────────────────────────────────────────────────── $cssBlock = @' <style> body { font-family: 'Segoe UI', Tahoma, Geneva, sans-serif; margin: 16px; color: #222; background: #f7f7f9; } h1 { color: #0078d4; margin-bottom: 4px; } h2 { color: #333; border-bottom: 2px solid #e1e1e1; padding-bottom: 4px; margin-top: 20px; } h3 { color: #444; margin-top: 14px; } table { border-collapse: collapse; width: 100%; background: #fff; margin-bottom: 12px; } th, td { border: 1px solid #d8d8d8; padding: 6px 10px; text-align: left; vertical-align: top; font-size: 13px; } th { background: #f0f4f8; color: #003366; } tr:nth-child(even) { background: #fafbfc; } .summary-card { background: #fff; border: 1px solid #d8d8d8; padding: 12px 16px; margin: 8px 0 14px 0; border-radius: 4px; box-shadow: 0 1px 3px rgba(0,0,0,0.05); } .tab-bar { display: flex; flex-wrap: wrap; gap: 4px; border-bottom: 2px solid #0078d4; margin-bottom: 0; } .tab-btn { background: #e1e1e1; border: 1px solid #c1c1c1; border-bottom: none; padding: 6px 14px; cursor: pointer; font-size: 13px; border-radius: 4px 4px 0 0; } .tab-btn.active { background: #fff; border-color: #0078d4; font-weight: bold; color: #0078d4; } .tab-panel { display: none; padding: 14px; background: #fff; border: 1px solid #c1c1c1; border-top: none; } .tab-panel.active { display: block; } .badge { display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 11px; font-weight: bold; } .badge-ok { background: #d4edda; color: #155724; border: 1px solid #c3e6cb; } .badge-warn { background: #fff3cd; color: #856404; border: 1px solid #ffeeba; } .badge-error { background: #f8d7da; color: #721c24; border: 1px solid #f5c6cb; } .badge-na { background: #e2e3e5; color: #383d41; border: 1px solid #d6d8db; } .footer { margin-top: 24px; color: #888; font-size: 11px; text-align: center; } .row-failed { background: #fdecea !important; } .row-warn { background: #fff8e1 !important; } /* Per-node result-row colouring — identical palette to the single-node report. */ tr.status-failed { background-color: #fde7e9 !important; } tr.status-success { background-color: #e6f4ea !important; } tr.status-skipped { background-color: #fff4ce !important; } .totals-card { background: #fff; border: 1px solid #d8d8d8; padding: 10px 16px; margin: 8px 0 14px 0; border-radius: 4px; box-shadow: 0 1px 3px rgba(0,0,0,0.05); } .totals-card .num { font-size: 18px; font-weight: bold; } .totals-card .ok { color: #155724; } .totals-card .fail { color: #721c24; } .totals-card .skip { color: #856404; } </style> '@ [void]$sb.AppendLine("<!DOCTYPE html>") [void]$sb.AppendLine("<html lang='en'><head><meta charset='utf-8'>") [void]$sb.AppendLine("<title>Cluster Connectivity Report — $(ConvertTo-HtmlSafe $ClusterResult.ClusterName)</title>") [void]$sb.AppendLine($cssBlock) [void]$sb.AppendLine("</head><body>") [void]$sb.AppendLine("<h1>Cluster Connectivity Report</h1>") [void]$sb.AppendLine("<div class='summary-card'>") [void]$sb.AppendLine("<strong>Cluster:</strong> $(ConvertTo-HtmlSafe $ClusterResult.ClusterName)<br>") [void]$sb.AppendLine("<strong>Orchestrator:</strong> $(ConvertTo-HtmlSafe $ClusterResult.OrchestratorMachine)<br>") [void]$sb.AppendLine("<strong>Run GUID:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.RunGuid.ToString()))<br>") [void]$sb.AppendLine("<strong>Start:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.StartTime.ToString('u')))<br>") [void]$sb.AppendLine("<strong>End:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.EndTime.ToString('u')))<br>") [void]$sb.AppendLine("<strong>Duration:</strong> $(ConvertTo-HtmlSafe ($ClusterResult.Duration.ToString()))<br>") [void]$sb.AppendLine("</div>") # ── Build per-node summary stats ───────────────────────────────────────── $nodeNames = @($ClusterResult.Nodes.Keys | Sort-Object) $summary = @{} foreach ($node in $nodeNames) { $rows = @($ClusterResult.Nodes[$node] | Where-Object { $_ -and $_.PSObject.Properties['Layer7Status'] -and $_.Layer7Status }) $tested = $rows.Count $passed = @($rows | Where-Object { $_.Layer7Status -eq 'Success' }).Count $failed = @($rows | Where-Object { $_.Layer7Status -eq 'Failed' }).Count $skipped = @($rows | Where-Object { $_.Layer7Status -eq 'Skipped' }).Count $sslInspected = @($rows | Where-Object { $_.Layer7Response -match 'SSL Inspection' }).Count $privateLink = @($rows | Where-Object { $_.Layer7Response -match 'Private Link|PrivateLink' }).Count $hasErrors = @($ClusterResult.Errors[$node]).Count -gt 0 $dlSpeed = if ($ClusterResult.PSObject.Properties['NodeDownloadSpeeds'] -and $ClusterResult.NodeDownloadSpeeds) { $ClusterResult.NodeDownloadSpeeds[$node] } else { $null } $status = if ($failed -gt 0 -or $hasErrors) { 'error' } elseif ($sslInspected -gt 0 -or $privateLink -gt 0) { 'warn' } else { 'ok' } $summary[$node] = [pscustomobject]@{ Tested = $tested Passed = $passed Failed = $failed Skipped = $skipped SSLInspected = $sslInspected PrivateLink = $privateLink DownloadSpeed = $dlSpeed Status = $status } } # ── Build the MERGED, flat result set (ComputerName as the FIRST column) ── # Used for the always-on JSON sidecar and the CSV output. ComputerName is set # to the node we fanned out to. The per-node HTML tabs intentionally OMIT this # column (the tab name + heading already identify the node). $mergedColumns = @( 'RowID','URL','Port','ArcGateway','IsWildcard','Source','IPAddress', 'Layer7Status','Layer7Response','Layer7ResponseTime','Note','TCPStatus', 'CertificateIssuer','CertificateSubject','CertificateThumbprint', 'IntermediateCertificateIssuer','IntermediateCertificateSubject','IntermediateCertificateThumbprint', 'RootCertificateIssuer','RootCertificateSubject','RootCertificateThumbprint' ) $mergedRows = [System.Collections.ArrayList]::new() foreach ($node in $nodeNames) { $nrows = @($ClusterResult.Nodes[$node] | Where-Object { $null -ne $_ }) foreach ($r in $nrows) { $ordered = [ordered]@{ ComputerName = $node } foreach ($c in $mergedColumns) { $ordered[$c] = if ($r.PSObject.Properties[$c]) { $r.$c } else { $null } } [void]$mergedRows.Add([pscustomobject]$ordered) } } # ── Rolled-up cluster totals (for the Summary tab + JSON) ───────────────── $totalTested = ($summary.Values | ForEach-Object { $_.Tested } | Measure-Object -Sum).Sum $totalPassed = ($summary.Values | ForEach-Object { $_.Passed } | Measure-Object -Sum).Sum $totalFailed = ($summary.Values | ForEach-Object { $_.Failed } | Measure-Object -Sum).Sum $totalSkipped = ($summary.Values | ForEach-Object { $_.Skipped } | Measure-Object -Sum).Sum if (-not $totalTested) { $totalTested = 0 } if (-not $totalPassed) { $totalPassed = 0 } if (-not $totalFailed) { $totalFailed = 0 } if (-not $totalSkipped) { $totalSkipped = 0 } # ── Tab bar (Summary + per-node) with status indicators ────────────────── [void]$sb.AppendLine('<div class="tab-bar">') [void]$sb.AppendLine('<button class="tab-btn active" onclick="showTab(event, ''tab-summary'')">Summary</button>') foreach ($node in $nodeNames) { $st = $summary[$node].Status $badge = switch ($st) { 'ok' { '<span class="badge badge-ok" style="font-size:9px;padding:1px 6px;">✓</span>' } 'warn' { '<span class="badge badge-warn" style="font-size:9px;padding:1px 6px;">⚠</span>' } 'error' { '<span class="badge badge-error" style="font-size:9px;padding:1px 6px;">✗</span>' } default { '' } } $safeNode = ConvertTo-HtmlSafe $node [void]$sb.AppendLine("<button class=`"tab-btn`" onclick=`"showTab(event, 'tab-$safeNode')`">$safeNode $badge</button>") } [void]$sb.AppendLine('</div>') # ── Summary tab — rolled-up totals + per-node Status Overview ──────────── [void]$sb.AppendLine('<div id="tab-summary" class="tab-panel active">') [void]$sb.AppendLine('<h2>Cluster Totals</h2>') [void]$sb.AppendLine('<div class="totals-card">') [void]$sb.AppendLine("<table style='width:auto;'><thead><tr><th>Nodes</th><th>Endpoints Tested</th><th>Successful</th><th>Failed</th><th>Skipped</th></tr></thead><tbody>") [void]$sb.AppendLine("<tr><td class='num'>$($nodeNames.Count)</td><td class='num'>$totalTested</td><td class='num ok'>$totalPassed</td><td class='num fail'>$totalFailed</td><td class='num skip'>$totalSkipped</td></tr>") [void]$sb.AppendLine('</tbody></table>') [void]$sb.AppendLine('</div>') [void]$sb.AppendLine('<h2>Per-Node Status Overview</h2>') [void]$sb.AppendLine('<table><thead><tr><th>Node</th><th>Tested</th><th>Successful</th><th>Failed</th><th>Skipped</th><th>SSL-Inspected</th><th>Private-Link</th><th>Download Speed</th><th>Status</th></tr></thead><tbody>') foreach ($node in $nodeNames) { $s = $summary[$node] $badgeClass = 'badge-' + $s.Status $badgeText = switch ($s.Status) { 'ok' { 'OK' } 'warn' { 'WARN' } 'error' { 'FAIL' } default { '?' } } $dlText = if ($s.PSObject.Properties['DownloadSpeed'] -and $s.DownloadSpeed) { ConvertTo-HtmlSafe ([string]$s.DownloadSpeed) } else { 'N/A' } [void]$sb.AppendLine("<tr><td>$(ConvertTo-HtmlSafe $node)</td><td>$($s.Tested)</td><td>$($s.Passed)</td><td>$($s.Failed)</td><td>$($s.Skipped)</td><td>$($s.SSLInspected)</td><td>$($s.PrivateLink)</td><td>$dlText</td><td><span class=`"badge $badgeClass`">$badgeText</span></td></tr>") } [void]$sb.AppendLine('</tbody></table>') # Errors block (if any) $totalErrors = ($ClusterResult.Errors.Values | ForEach-Object { @($_).Count } | Measure-Object -Sum).Sum if ($totalErrors -gt 0) { [void]$sb.AppendLine('<h2>Per-Node Errors</h2>') [void]$sb.AppendLine('<table><thead><tr><th>Node</th><th>Error</th></tr></thead><tbody>') foreach ($node in $nodeNames) { $errs = @($ClusterResult.Errors[$node]) foreach ($e in $errs) { [void]$sb.AppendLine("<tr class='row-failed'><td>$(ConvertTo-HtmlSafe $node)</td><td>$(ConvertTo-HtmlSafe $e)</td></tr>") } } [void]$sb.AppendLine('</tbody></table>') } [void]$sb.AppendLine('</div>') # ── Per-node tabs ──────────────────────────────────────────────────────── foreach ($node in $nodeNames) { $safeNode = ConvertTo-HtmlSafe $node [void]$sb.AppendLine("<div id=`"tab-$safeNode`" class=`"tab-panel`">") [void]$sb.AppendLine("<h2>Node: $safeNode</h2>") # Filter $null entries — a failed Invoke-Command leaves Nodes[<name>] = $null # and `@($null)` is a 1-element array containing $null (not @()), which would # otherwise blow up `$r.PSObject.Properties[...]` below. $rows = @($ClusterResult.Nodes[$node] | Where-Object { $null -ne $_ }) if ($rows.Count -eq 0) { [void]$sb.AppendLine("<p><em>No results returned from this node. See Errors on the Summary tab.</em></p>") } else { [void]$sb.AppendLine('<table><thead><tr>') # Per-node tab columns — full single-node column set, WITHOUT ComputerName # (the tab name + "Node:" heading already identify the host). $columns = $mergedColumns foreach ($c in $columns) { [void]$sb.AppendLine("<th>$c</th>") } [void]$sb.AppendLine('</tr></thead><tbody>') foreach ($r in $rows) { # Colour rows by Layer7Status using the SAME palette as the single-node # report: Success=green, Failed=red, Skipped=yellow. $statusVal = if ($r.PSObject.Properties['Layer7Status']) { [string]$r.Layer7Status } else { '' } $rowClass = switch ($statusVal) { 'Success' { 'status-success' } 'Failed' { 'status-failed' } 'Skipped' { 'status-skipped' } default { '' } } $cls = if ($rowClass) { " class=`"$rowClass`"" } else { '' } [void]$sb.AppendLine("<tr$cls>") foreach ($c in $columns) { $val = if ($r.PSObject.Properties[$c]) { $r.$c } else { '' } [void]$sb.AppendLine("<td>$(ConvertTo-HtmlSafe ([string]$val))</td>") } [void]$sb.AppendLine('</tr>') } [void]$sb.AppendLine('</tbody></table>') } [void]$sb.AppendLine('</div>') } # ── Footer + JS ───────────────────────────────────────────────────────── # Stamp the running module version into the footer so portable HTML artefacts # always record which build produced them (best-effort; 'version unknown' if unresolved). $footerModuleVersion = Get-LoadedModuleVersion -Name 'AzStackHci.DiagnosticSettings' [string]$footerModuleVersionLabel = if ($footerModuleVersion) { "v$($footerModuleVersion.ToString())" } else { 'version unknown' } [void]$sb.AppendLine(@" <div class="footer"> Generated by <strong>AzStackHci.DiagnosticSettings</strong> $footerModuleVersionLabel — Test-AzureLocalConnectivity -Scope Cluster — $(Get-Date -Format 'u') </div> <script> function showTab(evt, tabId) { var panels = document.querySelectorAll('.tab-panel'); for (var i = 0; i < panels.length; i++) panels[i].classList.remove('active'); var btns = document.querySelectorAll('.tab-bar > .tab-btn'); for (var i = 0; i < btns.length; i++) btns[i].classList.remove('active'); document.getElementById(tabId).classList.add('active'); evt.currentTarget.classList.add('active'); } </script> </body></html> "@) # ── Write the primary output file ──────────────────────────────────────── # HTML -> tabbed cluster report ($sb). CSV -> merged flat rows (ComputerName # first). JSON is ALWAYS written below in addition. (BOM-less UTF-8; PS 5.1 # Set-Content -Encoding UTF8 would add a BOM.) if ($OutputFormat -eq 'CSV') { try { # ConvertTo-Csv + Write-Utf8NoBom instead of Export-Csv -Encoding UTF8: the # latter prepends a UTF-8 BOM on PS 5.1 (the same BOM the comment above warns # about for the HTML path). @() guards the empty-rows case so an empty # collection binds cleanly to Write-Utf8NoBom's [string[]] parameter. $csvLines = @($mergedRows | ConvertTo-Csv -NoTypeInformation) Write-Utf8NoBom -Path $reportFile -Content $csvLines } catch { Write-Warning "Failed to write merged cluster CSV '$reportFile': $($_.Exception.Message)" } } else { Write-Utf8NoBom -Path $reportFile -Content $sb.ToString() } # ── JSON sidecar — full-fidelity cluster bundle (always written) ───────── # Schema v2: the per-node `Nodes` map is replaced by a single MERGED `Results` # array where every row carries ComputerName as its FIRST property, plus rolled # up `Totals`. Per-node summary/telemetry/errors are retained for drill-down. $nodeDurStrings = [ordered]@{} foreach ($node in $nodeNames) { $d = $ClusterResult.NodeDurations[$node] $nodeDurStrings[$node] = if ($d) { $d.ToString() } else { '' } } $orchestratorVer = (Get-LoadedModuleVersion -Name 'AzStackHci.DiagnosticSettings') $jsonObj = [ordered]@{ Schema = 'AzStackHciClusterConnectivity/v2' ModuleVersion = if ($orchestratorVer) { $orchestratorVer.ToString() } else { 'unknown' } ClusterName = $ClusterResult.ClusterName OrchestratorMachine = $ClusterResult.OrchestratorMachine RunGuid = $ClusterResult.RunGuid.ToString() StartTime = $ClusterResult.StartTime.ToString('u') EndTime = $ClusterResult.EndTime.ToString('u') Duration = $ClusterResult.Duration.ToString() Totals = [ordered]@{ Nodes = $nodeNames.Count Tested = $totalTested Successful = $totalPassed Failed = $totalFailed Skipped = $totalSkipped } NodeSummary = $summary NodeDurations = $nodeDurStrings NodeTelemetry = if ($ClusterResult.PSObject.Properties['NodeTelemetry']) { $ClusterResult.NodeTelemetry } else { @{} } Errors = $ClusterResult.Errors Results = $mergedRows.ToArray() } # Depth 8 is enough for: { Results: [ { flat row props } ] }. Write-Utf8NoBom -Path $jsonFile -Content ($jsonObj | ConvertTo-Json -Depth 8) return $reportFile } # SIG # Begin signature block # MIInRQYJKoZIhvcNAQcCoIInNjCCJzICAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCBBzB7UmvJpCNN9 # XLMHtXUqpZbvnmSaSOWaEqkEt1AUNqCCDLowggX1MIID3aADAgECAhMzAAACHU0Z # yE7XD1dIAAAAAAIdMA0GCSqGSIb3DQEBCwUAMFcxCzAJBgNVBAYTAlVTMR4wHAYD # VQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBD # b2RlIFNpZ25pbmcgUENBIDIwMjQwHhcNMjYwNDE2MTg1OTQzWhcNMjcwNDE1MTg1 # OTQzWjB0MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYD # VQQDExVNaWNyb3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IB # DwAwggEKAoIBAQDQvewXxx9gZZFC6Ys1WBay8BJ8kGA4JQnH5CMafqOASlTpK9H8 # o5ZXTXt0caVQTNMUPt445wXYD+dFtaKWTwDn1I52oUSrC9vJin1Gsqt+zyKJL5Dg # 3eQXbQNR61DmMy20GLTIO3SFed9Rfi/ophgCLGFLDR3r0KvHjwMb/jYWS0celV/4 # Lz27LfAekm8v9E5IXaeiXbAUYZKK090n4CVl3JBtbN+9DtI9SNu/yjvozW52/u7R # X/Ttpa/KDlpuokZ+Zcbvmtd9ur9gFLvZzh41o9MsE/clQtdaFWGvuo6Jua/ntpgk # ey3E5/vBFe+MJPG6phdnuo6r57ZudCudiI1bAgMBAAGjggGbMIIBlzAOBgNVHQ8B # Af8EBAMCB4AwHwYDVR0lBBgwFgYKKwYBBAGCN0wIAQYIKwYBBQUHAwMwHQYDVR0O # BBYEFH6QuMwqcPG0hQlQ6c5jCtTTLrVeMEUGA1UdEQQ+MDykOjA4MR4wHAYDVQQL # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xFjAUBgNVBAUTDTIzMDAxMis1MDc1NTkw # HwYDVR0jBBgwFoAUf1k/VCHarU/vBeXmo9ctBpQSCDEwYAYDVR0fBFkwVzBVoFOg # UYZPaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jcmwvTWljcm9zb2Z0 # JTIwQ29kZSUyMFNpZ25pbmclMjBQQ0ElMjAyMDI0LmNybDBtBggrBgEFBQcBAQRh # MF8wXQYIKwYBBQUHMAKGUWh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMv # Y2VydHMvTWljcm9zb2Z0JTIwQ29kZSUyMFNpZ25pbmclMjBQQ0ElMjAyMDI0LmNy # dDAMBgNVHRMBAf8EAjAAMA0GCSqGSIb3DQEBCwUAA4ICAQBKTbYOjzwTG/DXGaz9 # s6+fQeaTtDcFmMY+5UyVFCyj7Pv+5i37qfX8lSL/tBIfYQfWsMuBQlfZurJD6r4H # VJ2CeH+1fgiq8dcHdVKoZ3Sa2qXoX3cq9iS8cVb06B7+5/XJ7I0OxHH9fDsvJ3T3 # w5V/ZtAIFmLrl+P0CtG+92uzRsn0nTbdFjOkLMLWPLAU3THohKRlSEMgFJpPkm5n # 5UAZ35xX6FWCrDLsSKb555bTifwa8mJBwdlof0bmfYidH+dxZ1FdDxvLnNl9zeKs # A4kejaaIqqIPguhwAti5Ql7BlTNoJNwxCvBmqW2MQLnCkYN/VVUsR3V2x/rcTNzo # Bf/Z/SpROvdaA2ZOOd1uioXJt3tdLQ7vHpqpib0KfWr/FWXW10q38VxfCnRQBqzb # SuztR7nEMuzX7Ck+B/XaPDXd1qh72+QYyB0Z2VzWmO9zsnb9Uq/dwu8LGeQqnyu6 # 7SDGACvnXii2fb9+US492VTnXSnFKyqwgzUyFMtZK1/sHYTv6bG4TtQUygQxTN+Z # V+aJIlKO2MqZ7bKrAnOzS9m6NgoTdWOq11bTOZwKlIEV/EhV9SWkDmdpR/hPPT2v # 6TEj4F8PT/zHjRezIU5c/DGlt/VhY/pK0XkJtEyMmmS1BMtjU/rqBZVMIm3dnxQs # /TBByr+Cf8Z1r7aifQVQ+WSqzjCCBr0wggSloAMCAQICEzMAAAA5O7Y3Gb8GHWcA # AAAAADkwDQYJKoZIhvcNAQEMBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpX # YXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQg # Q29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRl # IEF1dGhvcml0eSAyMDExMB4XDTI0MDgwODIwNTQxOFoXDTM2MDMyMjIyMTMwNFow # VzELMAkGA1UEBhMCVVMxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEo # MCYGA1UEAxMfTWljcm9zb2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAyNDCCAiIwDQYJ # KoZIhvcNAQEBBQADggIPADCCAgoCggIBANgBnB7jOMeqlRYHNa265v4IY9fH8TKh # emHfPINe1gpLaV3dhg324WwH06LcHbpnsBukCDNitryo0dtS/EW6I/yEL/bLSY8h # KpbfQuWusBPr9qazYcDxCW/qnjb5JsI1s8bNOg3bVATvQVL4tcf03aTycsz8QeCd # M0l/yHRObJ9QqazM1r6VPEOJ7LL+uEEb73w6QCuhs89a1uv1zerOYMnsneRRwCbp # yW11IcggU0cRKDDq1pjVJzIbIF6+oiXXbReOsgeI8zu1FyQfK0fVkaya8SmVHQ/t # Of23mZ4W9k0Ri22QW9p3UgSC5OUDktKxxcCmGL6tXLfOGSWHIIV4YrTJTT6PNty5 # REojHJuZHArkF9VnHTERWoTjAzfI3kP+5b4alUdhgAZ7ttOu1bVnXfHaqPYl2rPs # 20ji03LOVWsh/radgE17es5hL+t6lV0eVHrVhsssROWJuz2MXMCt7iw7lFPG9LXK # Gjsmonn2gotGdHIuEg5JnJMJVmixd5LRlkmgYRZKzhxSCwyoGIq0PhaA7Y+VPct5 # pCHkijcIIDm0nlkK+0KyepolcqGm0T/GYQRMhHJlGOOmVQop36wUVUYklUy++vDW # eEgEo4s7hxN6mIbf2MSIQ/iIfMZgJxC69oukMUXCrOC3SkE/xIkgpfl22MM1itkZ # 35nNXkMolU1lAgMBAAGjggFOMIIBSjAOBgNVHQ8BAf8EBAMCAYYwEAYJKwYBBAGC # NxUBBAMCAQAwHQYDVR0OBBYEFH9ZP1Qh2q1P7wXl5qPXLQaUEggxMBkGCSsGAQQB # gjcUAgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU # ci06AjGQQ7kUBU7h6qfHMdEjiTQwWgYDVR0fBFMwUTBPoE2gS4ZJaHR0cDovL2Ny # bC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0MjAx # MV8yMDExXzAzXzIyLmNybDBeBggrBgEFBQcBAQRSMFAwTgYIKwYBBQUHMAKGQmh0 # dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0MjAx # MV8yMDExXzAzXzIyLmNydDANBgkqhkiG9w0BAQwFAAOCAgEAFJQfOChP7onn6fLI # MKrSlN1WYKwDFgAddymOUO3FrM8d7B/W/iQ6DxXsDn7D5W4wMwYeLystcEqfkjz4 # NURRgazyMu5yRzQh4LqjA4tStTcJh1opExo7nn5PuPBYnbu0+THSuVHTe0VTTPVh # ily/piFrDo3axQ9P4C+Ol5yet+2gTfekICS5xS+cYfSIvgn0JksVBVMYVI5QFu/q # hnLhsEFEUzG8fvv0hjgkO+lkpV9ty6GkN4vdnd7ya6Q6aR9y34aiM1qmxaxBi6OU # nyNl6fkuun/diTFnYDLTppOkr/mg5WSfCiDVMNCxtj4wPKC5OmHm1DQIt/MNokbb # H3UGsFP1QbzsLocuSqLCvH09Io3fDPTmscR9Y75G4qX7RTX8AdBPo0I6OEojf39z # uFZt0qOHm65YWQE69cZM2ueE1MB05dNNgHK9gTE7zKvK/fg8B2qjW88MT/WF5V5u # vZGtqa9FSL2RazArA+rDPuf6JGYz4HpgMZHB4S6szWSKYBv0VisCzfxgeU+dquXW # 9bd0auYlOB58DPcOYKdc3Se94g+xL4pcEhbB54JOgAkwYTu/9dLeH2pDqeJZAABV # DWRQCaXfO5LgyKwKCLYXpigrZYCjUSBcr+Ve8PFWMhVTQl0v4q8J/AUmQN5W4n10 # 1cY2L4A7GTQG1h32HHAvfQESWP0xghnhMIIZ3QIBATBuMFcxCzAJBgNVBAYTAlVT # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jv # c29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMjQCEzMAAAIdTRnITtcPV0gAAAAAAh0w # DQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYK # KwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEILqpjVtB # mMUGNIZdCBiwXE2K8/1UwWme0vbhTwrI4bP6MEIGCisGAQQBgjcCAQwxNDAyoBSA # EgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20w # DQYJKoZIhvcNAQEBBQAEggEAcdqthYDkhz8LrhaFhsZ6Nhhx08NeI5eqrlhWLMOn # lfFx3vamMPozQJfQ3JvhMc/CZIrUge1P5MTE4Vg71gQKRIZS9DgU6DrfImuSA8Na # EsCECPpqgYEN2LYcLD9u4hGFuHsn53i5I9C6NcpH6YxY2rZ2MlaXWZl2GqPHBQLd # /3vJuuam0Bs9c09NTmPzXOa0ORkgKDW/+HOIaN5/ZWGN6ayVFmfkThUWUCYNntwd # 4RdFL6AfOESeXflleFcqnOD2LXNeeGxWLtHvKSYz2kGC7Y958gdfYSPM3DVBHHOj # 9vT7DCktEakAvRz8SRQ7sBzQXAw4VyGqQ9QdF4DN4jm8UqGCF5MwghePBgorBgEE # AYI3AwMBMYIXfzCCF3sGCSqGSIb3DQEHAqCCF2wwghdoAgEDMQ8wDQYJYIZIAWUD # BAIBBQAwggFRBgsqhkiG9w0BCRABBKCCAUAEggE8MIIBOAIBAQYKKwYBBAGEWQoD # ATAxMA0GCWCGSAFlAwQCAQUABCChJ1FcT91fCS6JE6odVRvtFXHbNxXSEjuhvpSm # OnxdaQIGajGcsL1PGBIyMDI2MDcwNzE2NTYzMi41NlowBIACAfSggdGkgc4wgcsx # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJTAjBgNVBAsTHE1p # Y3Jvc29mdCBBbWVyaWNhIE9wZXJhdGlvbnMxJzAlBgNVBAsTHm5TaGllbGQgVFNT # IEVTTjozNzAzLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3Rh # bXAgU2VydmljZaCCEeowggcgMIIFCKADAgECAhMzAAACHzpwaeSiMC6VAAEAAAIf # MA0GCSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5n # dG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9y # YXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMB4X # DTI2MDIxOTE5Mzk1MVoXDTI3MDUxNzE5Mzk1MVowgcsxCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xJTAjBgNVBAsTHE1pY3Jvc29mdCBBbWVyaWNh # IE9wZXJhdGlvbnMxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVTTjozNzAzLTA1RTAt # RDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCCAiIw # DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMs7xcU5x8aoCqCLPT4CZCYWXd1n # RpMbhQUmSqo10wfLbwNqF4IGzo425+6TJ7nHjYJaTOBEjL2QTZVbASe1nxmSDKvK # QLRsiqwgPv7oXGC6x5Nd/VXC+bUPzXThWZ62gEmUni4Zu7IllS9cPHnmdWHnTKAt # PNnbhaRCyc+m9Fm/aQ9zf1/duEvIdW2cexr9b/zpWt+134B8W94D6o38Rj5caPlz # 8M8xcJgQJvRqthv3Z0Mla3DOnIGuniB8eWBjVQSlziXgAYQut/YnjCvFPNNb5Izx # eFXV044+tiMPTzQhtmovwH4gXREJ2fbr1hesYrpAgeKnOcplwJLyM3fRgAedMlU3 # lnOzq3/ZiyoEYOq68Np3v3fgUVPDO9Rw7dWgjJ33ddbC8/z9IIVUmHbVbygZBOm0 # YfKXL4WXiF6dUxVkXW/qiw62KtfwYVOISGd/ydF06DvJlgAnTHL0K0N9tdpOf9x/ # curc38YJgoWML7mZQIT4AmGbEy4x29JQaYqIAV2I8CNROqxZYEFkmbR4LCB4YkWa # ZAD5Xv/3wEpwT6BQvs715ZENDAp4By+jqvE2/ZjiMqscDpn/CLdr98pSEsI1kRLy # oZ2ukMCbuqH7oWNjHK0BuSIozq5M3L9Qs+XC2VhmgAkMNA/t5gLLDBVs1NsddEFJ # L41xwLSxIHhbtTrvAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQU2TvawYOUfSvkPC98 # ZHlfAkjwHtswHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYDVR0f # BFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jcmwv # TWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwGCCsG # AQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIwMjAx # MCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcDCDAO # BgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAGVu7cijKec/PQrWI9t4 # 2ex6PZvXmXmx2XYUAUEEZP2VF1zaA1XwsAi6w9gceFS9ENyzHiVsw2FUo8a7hMMt # qo238Ij5IW0a6p1EulU/VcT1wIvIqso+lwkUkKo+lX55+gC1gGYhRBzHHBPtYhDu # BqDz6uQq+syQKhGopLSYq/wnWwp+Lzn4ba6Fn/VG15JV1hk5k6P5JvjDOidMJOPs # S2Aw38Ffflbl1PN3vAl0Z6liRWLzvV1KsLZOvVkXMBHtLjh2sJZmknqmElptU06w # 3EUkqBLKS6A4ZbNDfXxGvcxM+DazcGez6lQ9WAyKN3htQ4fYGUSwswzA5yiVNNmq # DUdit1jWPGlQAj2KmMFBEg0v87vTln79/YuM2YlCigJUlVfbhp2lnnX1Kx9rMaip # ca33VuaoqjR8jT0iXixQeHiKHqumJAMGXIvu+a8J6PBXFh69jipXBNn+jeC+X5HS # UXFhL194gzg14bT5awNnuMtyLkwV643CixBjfPbpeDWiPRT276dxH25NT7EGYnwG # 2UJ2FDXdE0xfk/6StFg8HdcKn0mbpdo7X33mrfYAhmbWbMEYjrIeW+JdoQLlPMaI # 7Ute4+1dlTZf3ehAlsyh7e/z7kI8qtBqUZbJi6HZrdXWnBuP5bQUSYQU+m7xMj6p # g7UghBZRJG8WNe2Hk5vTaEwyMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJmQAA # AAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldh # c2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBD # b3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUg # QXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1WjB8 # MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVk # bW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1N # aWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEBBQAD # ggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjKNVf2 # AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhgfWpS # g0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJprx2r # rPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/dvI2k # 45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka97aSu # eik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKRHh09 # /SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9ituqBJR # 6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyOArxC # aC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItboKaD # IV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6bMUR # HXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6tAgMB # AAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQWBBQq # p1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacbUzUZ # 6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYzaHR0 # cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnkuaHRt # MBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBB # MAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2VsuP # 6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwubWlj # cm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEwLTA2 # LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93d3cu # bWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMu # Y3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/qXBS2 # Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6U03d # mLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVtI1Tk # eFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis9/kp # icO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTpkbKp # W99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0sHrY # UP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138eW0QB # jloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJsWkB # RH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7Fx0V # iY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0dFtq # 0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQtB1V # M1izoXBm8qGCA00wggI1AgEBMIH5oYHRpIHOMIHLMQswCQYDVQQGEwJVUzETMBEG # A1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWlj # cm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBP # cGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046MzcwMy0wNUUwLUQ5 # NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2WiIwoBATAH # BgUrDgMCGgMVAEsgyDU/uw24JemZsfYhdPa1d4QQoIGDMIGApH4wfDELMAkGA1UE # BhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAc # BgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0 # IFRpbWUtU3RhbXAgUENBIDIwMTAwDQYJKoZIhvcNAQELBQACBQDt9x8cMCIYDzIw # MjYwNzA3MDY0NTQ4WhgPMjAyNjA3MDgwNjQ1NDhaMHQwOgYKKwYBBAGEWQoEATEs # MCowCgIFAO33HxwCAQAwBwIBAAICGFAwBwIBAAICE7cwCgIFAO34cJwCAQAwNgYK # KwYBBAGEWQoEAjEoMCYwDAYKKwYBBAGEWQoDAqAKMAgCAQACAwehIKEKMAgCAQAC # AwGGoDANBgkqhkiG9w0BAQsFAAOCAQEAXDE60GKKD8og/4RKM38pTyvCP9JUEGqN # OSoQnRCCk9Rx52+8UAEsCrkl7v0YUo10ZVD66TClXi88onsNVGiCLe/+GvweoYwt # kxw/wRxqwY8a7p+2OTML36TPkCjK73Ej/VnvALsUmuEAJEDdI/UL4eq/w1pMtdQu # WRUnfb9DLJlkvN63m8ywpCJ7Z7RvK5lCbBycwhZ52sbr2QldQO2HL5ctDNKEcALi # gU2VbpgUP1Rr03TsuqZatrLfoOdmFDDafUNYOoZtyrwVjhjmHHgEs34n06PHvviF # WQ0KAF7ml8qTTzd2BWug7M1awtkj7rXmEEuP41EDKMYfXHllN0aZ9TGCBA0wggQJ # AgEBMIGTMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYD # VQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAk # BgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAACHzpwaeSi # MC6VAAEAAAIfMA0GCWCGSAFlAwQCAQUAoIIBSjAaBgkqhkiG9w0BCQMxDQYLKoZI # hvcNAQkQAQQwLwYJKoZIhvcNAQkEMSIEIEZBHx9Yt4vK+WBlnmMlzVVQqvgsxvTm # 7pM6O4md+HObMIH6BgsqhkiG9w0BCRACLzGB6jCB5zCB5DCBvQQgsCQK31aQKwy1 # RGQW7pNjQ/dRd1GcKJi49mF7fKQt/BQwgZgwgYCkfjB8MQswCQYDVQQGEwJVUzET # MBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMV # TWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1T # dGFtcCBQQ0EgMjAxMAITMwAAAh86cGnkojAulQABAAACHzAiBCA3LTZn4q1g5IfE # Tb2wlxb87pwV2s+2+y836wzBJR08azANBgkqhkiG9w0BAQsFAASCAgDEocKIO3nJ # KXEeXWISqUbgWfIx0kPrUkAChsg9r0AyvW+xpx0JXO0RP+RkzhNNWPtgU5mB+LnK # MU4bx1hbONLoZWa8LX0w6bKa9B8KvaYNC0zx8za105a5kA1ZLt31su6w2b4vnj5A # TdFbot9NVo3ZcsVaxWAOgh4O6nc4YJUwflkdnl8lY5eQ/Aiyd7dxoNI0u5B6dcZ7 # PnquBcZTk6B98qRZ+p8I15FRKFHdgzCQDY6dTYYEeW4rjTLyQjDoRMcS4qji6Fhe # cBYgORej/XshcWx0ffeTsny5S+EaXvguYnE/hlUq7pMF7CRHrblahREIFH5BfM2+ # G3fqb3xHR4m+GUz4NjdWjVShuUGDR3m2wdC0nNVLFai4JYnOh6UmGDkqqgjEMd5Z # N9WLZmIxgH0Ae2NgR0R22sm5njKlHeWk89yqdi7hHeB4JA3IWbio2t4amDvTkB1q # SEegVV4zYowxushqNGyTNiHvMnt7JW3PX2AWCj1oQH+YFDP1bgGj+9ynCVePrxEB # /rpCxcL9zj72IiL2cXjwC0tINaRv2jmcferOD9ETL+eSGFVq3QbmV7OHd5QqHdyi # A7OOZdi+D4LLzsoxju1OWMDStaLm7G6PgaeLtRTVAm0OsZ2wMYbhOH4s70jYV+4A # uqiqNyO/bkSxCzlHeEtuupZKvny52cvdPA== # SIG # End signature block |