AzStackHciLLDP/AzStackHci.LLDP.Helpers.psm1
Import-LocalizedData -BindingVariable lnTxt -FileName AzStackHci.LLDP.Strings.psd1 function New-PsSessionWithRetriesInternal { param ( [System.String] $Node, [PSCredential] $Credential, [System.Int16] $Retries = 60, [System.Int16] $WaitSeconds = 10 ) for ($i=1; $i -le $Retries; $i++) { try { Trace-Execution "Creating PsSession ($i/$Retries) to $Node as $($Credential.UserName)..." $psSessionCreated = Microsoft.PowerShell.Core\New-PSSession -ComputerName $Node -Credential $Credential -ErrorAction Stop $computerNameFromSession = Microsoft.PowerShell.Core\Invoke-Command -Session $psSessionCreated -ScriptBlock { $ENV:COMPUTERNAME } -ErrorAction Stop $isAdminSession = Microsoft.PowerShell.Core\Invoke-Command -Session $psSessionCreated -ScriptBlock { ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] 'Administrator') } -ErrorAction Stop if (-not $isAdminSession) { throw ("PsSession was successful but user: {0} is not an administrator on computer {1} " -f $psSessionCreated.Runspace.ConnectionInfo.Credential.Username, $computerName) } break } catch { Trace-Execution "Creating PsSession ($i/$Retries) to $Node failed: $($_.exception.message)" $errMsg = $_.tostring() Start-Sleep -Seconds $WaitSeconds } } if ($psSessionCreated -and $computerNameFromSession -and $isAdminSession) { Trace-Execution ("PsSession to {0} created after {1} retries. (Remote machine name: {2})" -f $Node, ("$i/$retries"), $computerNameFromSession) return $psSessionCreated } else { throw "Unable to create a valid session to $Node`: $errMsg" } } function EnsureTestSessionOpen { <# .SYNOPSIS Make sure the test session is opened for the given PSSessions .DESCRIPTION Make sure the test session is opened for the given PSSessions. If the session is not opened, open a new session for it. .PARAMETER PSSessions The PSSessions to be checked .EXAMPLE EnsureTestSessionOpen -PSSessions $PSSessions #> [CmdletBinding()] param ( [System.Management.Automation.Runspaces.PSSession[]] $PSSessions ) [System.Management.Automation.Runspaces.PSSession[]] $newTestSessionsAfterChecking = @() foreach ($testSession in $PSSessions) { [System.Management.Automation.Runspaces.PSSession] $sessionToReturn = $null Log-Info "[EnsureTestSessionOpen] Clean up PSSession on $($testSession.ComputerName) and create a new session" Remove-PSSession -Session $testSession -ErrorAction SilentlyContinue $sessionCredential = $testSession.Runspace.ConnectionInfo.Credential $sessionToReturn = New-PsSessionWithRetriesInternal -Node $testSession.ComputerName -Credential $sessionCredential $newTestSessionsAfterChecking += $sessionToReturn } return $newTestSessionsAfterChecking } function GetLLDPNbrTLVs { [CmdletBinding()] param ( [string] $ComputerName ) # Check if the NetLldpAgent module is available if (-not (Get-Command -Name Get-NetLldpAgent -ErrorAction SilentlyContinue)) { throw $lnTxt.NoNetLldpAgentModule -f $ComputerName } # Initialize the return object $retVal = [PSCustomObject]@{ Pass = $true Message = "Check LLDP neighbor TLVs on Host $ComputerName" } # Initialize the LLDP neighbor TLVs hashtable $NbrLLDPTLVs = @{} $HostAdapter = @{} # Get the list of physical network adapters $adapterList = Get-NetAdapter -Physical | Where-Object { $_.Status -eq 'Up' -and $_.Name -match 'ethernet' } foreach ($Adapter in $adapterList) { # Ensure the NetLldpAgent is enabled for all physical adapters Enable-NetLLDPAgent -NetAdapterName $Adapter.Name # Get the LLDP neighbor information for each adapter $NbrLLDPObj = (Get-NetLldpAgent -NetAdapterName $Adapter.Name | Where-Object { $_.Scope -eq 'NearestBridge' }).Neighbor # Get the Host adapter information for each adapter # Filter Host Adapter Attributes Here $HostAdapterObj = Get-NetAdapter -Name $Adapter.Name | Select-Object Name, InterfaceDescription, DriverInformation, Status, MacAddress, LinkSpeed if ($null -eq $NbrLLDPObj) { $retVal.Pass = $false $retVal.Message += "`n" + $lnTxt.NoNetLldpAgentModule -f $Adapter.Name } else { $NbrLLDPTLVs[$Adapter.Name] = $NbrLLDPObj.Tlvs } if ($null -eq $HostAdapterObj) { $retVal.Pass = $false $retVal.Message += "`n" + $lnTxt.NoHostAdapterTlvs -f $Adapter.Name } else { $HostAdapter[$Adapter.Name] = $HostAdapterObj } } return $retVal, $NbrLLDPTLVs, $HostAdapter } function Test-LLDPNbrTlvs { [CmdletBinding()] param ( [System.Management.Automation.Runspaces.PSSession[]] $PSSession, [string] $OutputPath ) try { # Ensure all test sessions are open [System.Management.Automation.Runspaces.PSSession[]] $allNodeSessions = EnsureTestSessionOpen -PSSessions $PSSession # Inistialize the results array $LLDPTestResults = @() $LLDPNbrTestStatus = 'SUCCESS' $LLDPNbriDetailMsg = "Check LLDP Neighbor TLVs on Hosts" foreach ($testSession in $allNodeSessions) { # Make sure host has LLDP neighbors Log-Info "Check LLDP Neighbor on Host [ $($testSession.ComputerName) ]" $tmpCheckRst, $tmpNbrLldpTLVs, $tmpHostAdapter = Invoke-Command -Session $testSession -ScriptBlock ${function:GetLLDPNbrTLVs} -ArgumentList @(, $testSession.ComputerName) if (-not $tmpCheckRst.Pass) { $LLDPNbrTestStatus = 'FAILURE' $LLDPNbriDetailMsg += $tmpCheckRst.Message } #region Save the LLDP Neighbor TLVs per session to a JSON file if ($tmpNbrLldpTLVs.Count -eq 0) { Log-Info "No LLDP Neighbor TLVs found for $($testSession.ComputerName)" continue } $NbrLldpJsonPath = "$OutputPath\NBRLLDPTLV_$($testSession.ComputerName).json" $HostAdapterJsonPath = "$OutputPath\HOSTADAPTER_$($testSession.ComputerName).json" # Convert the hashtable to JSON and save it to a file $tmpNbrLldpTLVs | ConvertTo-Json | Set-Content -Path $NbrLldpJsonPath -Encoding utf8 $tmpHostAdapter | ConvertTo-Json | Set-Content -Path $HostAdapterJsonPath -Encoding utf8 Log-Info "LLDP Neighbor TLVs saved to $NbrLldpJsonPath" Log-Info "LLDP Host TLVs saved to $HostAdapterJsonPath" #endregion } $LLDPNbrCheckRstObject = @{ Name = 'AzStackHci_LLDP_Test_Neighbor_Existance' Title = 'Validate Host LLDP Neighbor Existence' DisplayName = 'Validate Host LLDP Neighbor Existence' Severity = 'WARNING' Description = 'Check if all hosts have LLDP neighbors' Tags = @{} Remediation = $lnTxt.NoNetLldpAgentModule.TestLLDPNbrTlvsRemidation TargetResourceID = 'ValidateLLDPNeighborExistence' TargetResourceName = 'ValidateLLDPNeighborExistence' TargetResourceType = 'ValidateLLDPNeighborExistence' Timestamp = [datetime]::UtcNow Status = $LLDPNbrTestStatus AdditionalData = @{ Source = 'AllHosts' Resource = 'ValidateLLDPNeighborExistence' Detail = $LLDPNbriDetailMsg Status = $LLDPNbrTestStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $LLDPTestResults += New-AzStackHciResultObject @LLDPNbrCheckRstObject return $LLDPTestResults } catch { throw "An error occurred while running Test-LLDPNbrTlvs: $_" } finally { # Final log to indicate the end of the process Log-Info "Completed Test-LLDPNbrTlvs for $($PSSession.ComputerName)" } } function Convert-TlvChassisIdData { param ( [string] $TLVByteString ) # Convert the string of byte values to an array of bytes $TLVBytes = $TLVByteString -split ' ' | ForEach-Object { [byte]$_ } # Check if $TLVBytes is null or empty if (-not $TLVBytes -or $TLVBytes.Length -le 1) { return "Unknown" } # Get the Chassis ID Subtype from the first byte [int]$chasIdType = $TLVBytes[0] # Switch statement to handle different Chassis ID Subtypes switch ($chasIdType) { 4 { # Format the remaining bytes as a MAC address return ("{0:X2}:{1:X2}:{2:X2}:{3:X2}:{4:X2}:{5:X2}" -f $TLVBytes[1..($TLVBytes.Length - 1)]) } 7 { # Type 7 indicates a locally assigned chassis ID, interpret remaining bytes as ASCII return ([System.Text.Encoding]::ASCII.GetString($TLVBytes[1..($TLVBytes.Length - 1)])) } Default { # Log a warning for other or unknown Chassis ID Subtypes Write-Warning "Unknown Chassis ID TLV: $($TLVBytes -join ', ')" return "Unknown" } } } function Convert-TlvSystemNameData { param ( [string] $TLVByteString ) # Convert the string of byte values to an array of bytes $TLVBytes = $TLVByteString -split ' ' | ForEach-Object { [byte]$_ } # Check if $TLVBytes is null or has a length of 1 or less if (-not $TLVBytes -or $TLVBytes.Length -le 1) { return "Unknown" } # Convert the byte array to a string, skipping the first byte return [System.Text.Encoding]::ASCII.GetString($TLVBytes[0..($TLVBytes.Length - 1)]) } function Convert-TlvSystemDescData { param ( [string] $TLVByteString ) # Convert the string of byte values to an array of bytes $TLVBytes = $TLVByteString -split ' ' | ForEach-Object { [byte]$_ } # Check if $TLVBytes is null or has a length of 1 or less if (-not $TLVBytes -or $TLVBytes.Length -le 1) { return "Unknown" } # Convert the byte array to a string, skipping the first byte return [System.Text.Encoding]::ASCII.GetString($TLVBytes) } function Convert-TlvPortIdData { param ( [string] $TLVByteString ) # Convert the string of byte values to an array of bytes $TLVBytes = $TLVByteString -split ' ' | ForEach-Object { [byte]$_ } if(-not $TLVBytes -or $TLVBytes.Length -le 1 ){ return "Unknown" } # get the Chassis ID Subtype [int]$portIdType = "0x$($TLVBytes[0])" switch -Regex ($portIdType) { "[1-2]|[5]"{return ( [System.Text.Encoding]::ASCII.GetString($TLVBytes[1..($TLVBytes.Length - 1)]) )} # Type 3 is a MAC address "3"{return ("{0:X2}:{1:X2}:{2:X2}:{3:X2}:{4:X2}:{5:X2}" -f $TLVBytes[1..($TLVBytes.Length - 1)])} Default { Write-Warning "Unknown Port ID TLV: $($TLVBytes -join ', ')" return "Unknown" } } } function Convert-TlvVLANIdData{ param ( [string] $TLVByteString ) # Convert the string of byte values to an array of bytes $TLVBytes = $TLVByteString -split ' ' | ForEach-Object { [byte]$_ } if(-not $TLVBytes -or $TLVBytes.Length -le 1 ){ return "Unknown" } $list = @() foreach($i in $TLVBytes){ #Bitshift the first byte by 8, add the second byte $list+= [convert]::ToSingle($i.data[0]) * [math]::Pow(2,8) + [convert]::ToSingle($i.data[1]) } return $list } function Convert-TlvMaxFrameSize { param ( [string] $TLVByteString ) # Convert the string of byte values to an array of bytes $TLVBytes = $TLVByteString -split ' ' | ForEach-Object { [byte]$_ } if(-not $TLVBytes -or $TLVBytes.Length -lt 2){ return "Unknown" } # Calculate the decimal value $MaxFrameValue = ($TLVBytes[0] * 256) + $TLVBytes[1] return [int]$MaxFrameValue } function Convert-TlvNativeVLAN{ param ( [string] $TLVByteString ) # Convert the string of byte values to an array of bytes $TLVBytes = $TLVByteString -split ' ' | ForEach-Object { [byte]$_ } if(-not $TLVBytes -or $TLVBytes.Length -lt 2){ return "Unknown" } $NativeVlanValue = ($TLVBytes[0] * 256) + $TLVBytes[1] return [int]$NativeVlanValue } function Convert-TlvVLANIdData{ param ( [object[]] $TLVByteStringList ) if(-not $TLVByteStringList -or $TLVByteStringList.Length -le 1 ){ return "Unknown" } $VlanIdList = @() foreach($TLVByteString in $TLVByteStringList){ $tlvArray = $TLVByteString -split ' ' | ForEach-Object { [int]$_ } $vlanIdHex = "{0:X2}{1:X2}" -f $tlvArray[0], $tlvArray[1] $vlanIdDecimal = [convert]::ToInt32($vlanIdHex, 16) # Convert hex to decimal $VlanIdList+= $vlanIdDecimal } return $VlanIdList } # Function to convert TLV ETS configuration data from a byte string to an array of bytes function Convert-TlvETSConfigurationData { param ( [string] $TLVByteString # Input string containing byte values separated by spaces ) # Validate the input to ensure it's not null or empty if (-not $TLVByteString) { return "Unknown" } # Convert the string of byte values into an array of bytes try { $TLVBytes = $TLVByteString -split ' ' | ForEach-Object { [byte]$_ } return $TLVBytes } catch { # Return "Unknown" if there's an error in the conversion return "Unknown" } } # Function to convert TLV PFC configuration data from a byte string to an array of bytes function Convert-TlvPFCConfigurationData { param ( [string] $TLVByteString # Input string containing byte values separated by spaces ) # Validate the input to ensure it's not null or empty if (-not $TLVByteString) { return "Unknown" } # Convert the string of byte values into an array of bytes try { $TLVBytes = $TLVByteString -split ' ' | ForEach-Object { [byte]$_ } return $TLVBytes } catch { # Return "Unknown" if there's an error in the conversion return "Unknown" } } # Function to merge adapter data from two hashtables function Merge-AdapterData { param ( [hashtable]$existingData, # Existing hashtable that will be merged into [hashtable]$newData # New hashtable containing data to merge ) foreach ($key in $newData.Keys) { if ($existingData.ContainsKey($key)) { $existingData[$key] += $newData[$key] } else { $existingData[$key] = $newData[$key] } } } # Core function to export merged LLDP data to a JSON file function Export-MergedLLDPDataToJson { param ( [string] $OutputPath, # Path where the output JSON file will be saved [string] $NbrFilterString = 'NBRLLDPTLV_*.json', # Filter string to locate neighbor LLDP TLV JSON files [string] $HostFilterString = 'HOSTADAPTER_*.json' # Filter string to locate host adapter JSON files ) # Set the output file path for the merged LLDP JSON file $outputFilePath = Join-Path -Path $OutputPath -ChildPath "MergedLLDPData.json" # Search for matching Neighbor LLDP TLV JSON files $nbrLldpJsonFiles = Get-ChildItem -Path $OutputPath -Filter $NbrFilterString # Search for matching Host Adapter JSON files $hostAdapterJsonFiles = Get-ChildItem -Path $OutputPath -Filter $HostFilterString # Check if any Neighbor LLDP TLV JSON files are found if ($nbrLldpJsonFiles.Length -eq 0) { throw "Error: No Neighbor LLDP TLV JSON files found at '$OutputPath'. Neighbor devices may not have sent or the host may not have received LLDP packets correctly. Please check logs for more details." } # Check if any Host Adapter JSON files are found if ($hostAdapterJsonFiles.Length -eq 0) { throw "Error: No Host adapter JSON files found at '$OutputPath'. The host may not have sent LLDP packets properly. Please check logs for more details." } # Initialize the hashtable to store the merged LLDP data $mergedLLDPJson = @{} # Iterate through all Neighbor LLDP JSON files and process them foreach ($file in $nbrLldpJsonFiles) { $jsonContent = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json $hciHost = $jsonContent.PSComputerName # Prepare adapter data from the neighbor LLDP TLVs $adapterData = @{} foreach ($key in $jsonContent.PSObject.Properties.Name) { if ($key -notmatch 'ethernet') { continue } # Skip non-ethernet keys $adapterName = $key $tlvListObj = $jsonContent.$key $adapterData[$adapterName] = [ordered]@{ "RemoteChassisID" = Convert-TlvChassisIdData ($tlvListObj | Where-Object TLVType -eq 1).data "RemoteSystemName" = Convert-TlvSystemNameData ($tlvListObj | Where-Object TLVType -eq 5).data "RemoteSystemDesc" = Convert-TlvSystemDescData ($tlvListObj | Where-Object TLVType -eq 6).data "RemotePortID" = Convert-TlvPortIdData ($tlvListObj | Where-Object TLVType -eq 2).data "RemoteMaxFrameSize" = Convert-TlvMaxFrameSize ($tlvListObj | Where-Object { $_.TLVType -eq 127 -and $_.OuiSubtype -eq 4 }).data "RemoteNativeVLAN" = Convert-TlvNativeVLAN ($tlvListObj | Where-Object { $_.TLVType -eq 127 -and $_.OuiSubtype -eq 1 -and $_.Oui -eq "0 128 194" }).data "RemoteVLANIDs" = Convert-TlvVLANIdData ($tlvListObj | Where-Object { $_.TLVType -eq 127 -and $_.OuiSubtype -eq 3 }).data "RemoteETS" = Convert-TlvETSConfigurationData ($tlvListObj | Where-Object { $_.TLVType -eq 127 -and $_.OuiSubtype -eq 9 }).data "RemotePFC" = Convert-TlvPFCConfigurationData ($tlvListObj | Where-Object { $_.TLVType -eq 127 -and $_.OuiSubtype -eq 11 }).data } } # Merge the adapter data into the host entry if (-not $mergedLLDPJson.ContainsKey($hciHost)) { $mergedLLDPJson[$hciHost] = @{} } Merge-AdapterData -existingData $mergedLLDPJson[$hciHost] -newData $adapterData } # Iterate through all Host Adapter JSON files and process them foreach ($file in $hostAdapterJsonFiles) { $jsonContent = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json $hciHost = $jsonContent.PSComputerName # Prepare adapter data from the host JSON files $adapterData = @{} foreach ($key in $jsonContent.PSObject.Properties.Name) { if ($key -notmatch 'ethernet') { continue } # Skip non-ethernet keys $adapterName = $key $adapterObj = $jsonContent.$key $adapterData[$adapterName] = [ordered]@{ "LocalAdapterName" = $adapterObj.Name "LocalAdapterDescription" = $adapterObj.InterfaceDescription "LocalAdapterDriverInformation" = $adapterObj.DriverInformation "LocalAdapterStatus" = $adapterObj.Status "LocalAdapterMacAddress" = $adapterObj.MacAddress "LocalAdapterLinkSpeed" = $adapterObj.LinkSpeed } } # Merge the adapter data into the host entry if (-not $mergedLLDPJson.ContainsKey($hciHost)) { $mergedLLDPJson[$hciHost] = @{} } Merge-AdapterData -existingData $mergedLLDPJson[$hciHost] -newData $adapterData } # Convert the merged LLDP data to JSON and save it to the output file $mergedLLDPJson | ConvertTo-Json -Depth 10 | Set-Content -Path $outputFilePath # Return the path to the merged LLDP JSON file return $outputFilePath } # Function to test the generation of merged LLDP data to JSON file function Test-MergedLLDPDataToJson { param ( [string] $OutputPath # Path to the output directory where the merged LLDP JSON file will be created ) # Generate the merged LLDP JSON file by calling the Export function $mergedLLDPJson = Export-MergedLLDPDataToJson -OutputPath $OutputPath # Verify if the merged JSON file was successfully created if (-not (Test-Path -Path $mergedLLDPJson)) { throw $lnTxt.NoMergedLLDPJson -f $mergedLLDPJson } # Initialize the return object to store the test result $retVal = [PSCustomObject]@{ Pass = $true # Indicates whether the test passed or failed Message = "Successfully generated the merged LLDP JSON file." } # Parse the JSON content from the generated file $mergedLLDPJsonObj = Get-Content -Path $mergedLLDPJson | ConvertFrom-Json # Check if the JSON file contains any data if ($mergedLLDPJsonObj.Count -eq 0) { $retVal.Pass = $false $retVal.Message += "`n" + $lnTxt.NoMergedLLDPJsonObj -f $mergedLLDPJson } # Set the final test status based on the presence of data $LLDPMergedJSONTestStatus = if ($retVal.Pass) { 'SUCCESS' } else { 'FAILURE' } $LLDPMergedJSONDetailMsg = $retVal.Message # Create the result object to store test results $LLDPMergedJsonRstObject = @{ Name = 'AzStackHci_LLDP_Test_Merged_LLDP_To_Json' Title = 'Export Merged LLDP to JSON' DisplayName = 'Export Merged LLDP to JSON' Severity = 'WARNING' Description = "Convert and generate the merged LLDP JSON file located at: $mergedLLDPJson." Tags = @{ } Remediation = $lnTxt.GenerateMergedLLDPJsonRemidation TargetResourceID = 'AllHosts' TargetResourceName = 'GenerateMergedLLDPJson' TargetResourceType = 'GenerateMergedLLDPJson' Timestamp = [datetime]::UtcNow Status = $LLDPMergedJSONTestStatus AdditionalData = @{ Source = 'AllHosts' Resource = 'GenerateMergedLLDPJson' Detail = $LLDPMergedJSONDetailMsg Status = $LLDPMergedJSONTestStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } # Add the result object to the results array and return it $LLDPTestResults += New-AzStackHciResultObject @LLDPMergedJsonRstObject return $LLDPTestResults } # Function to validate LLDP connections between hosts and switches function Test-LLDPConnections { [CmdletBinding()] param ( [string] $OutputPath, # Path to the output directory where the LLDP JSON file is located [array] $PhysicalNodeList # Parameter for physical node list ) try { # Construct the path for the merged LLDP JSON file $LLDPJsonFile = Join-Path -Path $OutputPath -ChildPath "MergedLLDPData.json" # Check if the JSON file exists if (-Not (Test-Path -Path $LLDPJsonFile)) { throw "Error: The file '$LLDPJsonFile' does not exist. Please verify the file path." } # Parse the JSON file to retrieve LLDP connection data $LLDPJson = Get-Content -Path $LLDPJsonFile | ConvertFrom-Json # Initialize an array to store the connection results $connections = @() #Create name to IP mapping from answer file if ($PhysicalNodeList -and $PhysicalNodeList.Count -gt 0) { $name2IpPath = Join-Path $OutputPath 'NodeName2Ip.json' $name2Ip = @{} foreach ($n in $PhysicalNodeList) { $name2Ip[$n.name] = $n.ipv4Address } $name2Ip | ConvertTo-Json | Set-Content -Encoding utf8 $name2IpPath } # Iterate through each host in the JSON data foreach ($node in $LLDPJson.PSObject.Properties) { $nodeName = $node.Name # Host name $nodeObject = $node.Value.PSObject.Properties # Host properties # Loop through each adapter for the current host foreach ($adapter in $nodeObject) { $a = $adapter.Value if ([string]::IsNullOrEmpty($a.RemoteSystemName) -or [string]::IsNullOrEmpty($a.RemoteChassisID) -or [string]::IsNullOrEmpty($a.RemotePortID)) { if (-not $missingTlvFound) { $warnObj = @{ Name = 'AzStackHci_Hosts_Missing_LLDP_TLVs_To_Validated_Connections' Title = 'Missing LLDP TLVs for Validated Connections' DisplayName = 'Missing LLDP TLVs for Validated Connections' Severity = 'WARNING' Description = $lnTxt.UnknownLLDPNeighbor Tags = @{} Remediation = $lnTxt.TestLLDPNbrTlvsRemidation TargetResourceID = 'MissingLLDPTLVsforValidatedConnections' TargetResourceName = 'MissingLLDPTLVsforValidatedConnections' TargetResourceType = 'MissingLLDPTLVsforValidatedConnections' Timestamp = [datetime]::UtcNow Status = 'FAILURE' AdditionalData = @{ Source = 'MergedLLDPData.json' Resource = 'LLDPConnectionsValidated' Detail = $lnTxt.MissingLLDPConnectionsDetail Status = 'FAILURE' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $LLDPConnectionResults += New-AzStackHciResultObject @warnObj } $missingTlvFound = $true continue } $connections += [PSCustomObject]@{ LocalHostName = $nodeName LocalAdapterName = $a.LocalAdapterName LocalAdapterDescription = $a.LocalAdapterDescription LocalMacAddress = $a.LocalAdapterMacAddress RemoteSystemName = $a.RemoteSystemName RemoteChassisID = $a.RemoteChassisID.Substring(0, $a.RemoteChassisID.Length - 3) RemotePortId = $a.RemotePortID } } } # Save the connections array to a JSON file $connectionJsonFile = Join-Path -Path $OutputPath -ChildPath "Connections.json" $connections | ConvertTo-Json -Depth 5 | Out-File -FilePath $connectionJsonFile -Encoding utf8 Log-Info "Full connection map saved to $connectionJsonFile" $LLDPConnectionStatusMessage = "Host LLDP Connections:" # Validation 1: Ensure that there are connections present if ($connections.Count -le 0) { $LLDPConnectionStatusMessage += "`n" + $lnTxt.NoLLDPConnectionsFound Log-Info $LLDPConnectionStatusMessage -Type 'WARNING' $LLDPConnectionRstObject = @{ Name = "AzStackHci_Hosts_Have_No_LLDP_Connections" Title = 'No LLDP Connections Detected' DisplayName = 'No LLDP Connections Detected' Severity = 'Warning' Description = $lnTxt.NoLLDPConnectionsFound Tags = @{ } Remediation = $lnTxt.NoLLDPConnectionsFoundRemidation TargetResourceID = 'NoLLDPConnectionsFound' TargetResourceName = 'NoLLDPConnectionsFound' TargetResourceType = 'NoLLDPConnectionsFound' Timestamp = [datetime]::UtcNow Status = 'FAILURE' AdditionalData = @{ Source = 'MergedLLDPData.json' Resource = 'LLDPConnectionsValidated' Detail = $LLDPConnectionStatusMessage Status = 'FAILURE' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $LLDPConnectionResults += New-AzStackHciResultObject @LLDPConnectionRstObject return $LLDPConnectionResults } # Validation 2: Ensure that each host is connected to the same number of network switches $node2SwitchDict = @{} foreach ($connection in $connections) { $nodeKey = $connection.LocalHostName $connectItem = [PSCustomObject]@{ HostAdapterName = $connection.LocalAdapterName HostAdapterDescription = $connection.LocalAdapterDescription HostMacAddress = $connection.LocalMacAddress SwitchPortId = $connection.RemotePortId } $switchKey = $connection.RemoteSystemName + "_" + $connection.RemoteChassisID if ($node2SwitchDict.ContainsKey($nodeKey)) { if ($node2SwitchDict[$nodeKey].ContainsKey($switchKey)) { $node2SwitchDict[$nodeKey][$switchKey] += $connectItem } else { $node2SwitchDict[$nodeKey][$switchKey] = @($connectItem) } } else { $node2SwitchDict[$nodeKey] = @{ $switchKey = @($connectItem) } } } $node2SwitchJsonFile = Join-Path -Path $OutputPath -ChildPath "Node2Switch.json" $node2SwitchDict | ConvertTo-Json -Depth 5 | Out-File -FilePath $node2SwitchJsonFile -Encoding utf8 Log-Info "Connection map saved to $node2SwitchJsonFile" $firstKey = $node2SwitchDict.Keys | Select-Object -First 1 $expectedLength = $node2SwitchDict[$firstKey].Count foreach ($nodeKey in $node2SwitchDict.Keys) { $length = $node2SwitchDict[$nodeKey].Count if ($length -ne $expectedLength) { $LLDPConnectionStatusMessage += "`n Host '$nodeKey' expects to be connected to $expectedLength switches but is connected to $length." Log-Info $LLDPConnectionStatusMessage -Type 'WARNING' $LLDPConnectionRstObject = @{ Name = "HCI_Node_Connect_Different_Number_of_Network_Device" Title = 'Host Connected to a Different Number of Switches' DisplayName = 'Host Connected to a Different Number of Switches' Severity = 'WARNING' Description = "Host '$nodeKey' expects to be connected to $expectedLength switches but is connected to $length." Tags = @{ } Remediation = $lnTxt.ConnectionMismatchRemidation TargetResourceID = 'HostSwitchConnectionMismatch' TargetResourceName = 'HostSwitchConnectionMismatch' TargetResourceType = 'HostSwitchConnectionMismatch' Timestamp = [datetime]::UtcNow Status = 'FAILURE' AdditionalData = @{ Source = 'Connections.json' Resource = 'LLDPConnectionsValidated' Detail = $LLDPConnectionStatusMessage Status = 'FAILURE' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $LLDPConnectionResults += New-AzStackHciResultObject @LLDPConnectionRstObject return $LLDPConnectionResults } } # Validation 3: Ensure that each network switch is connected to the same number of hosts $switch2NodeDict = @{} foreach ($connection in $connections) { $switchKey = "$($connection.RemoteSystemName)_$($connection.RemoteChassisID)" $connectItem = [PSCustomObject]@{ HostAdapterName = $connection.LocalAdapterName HostAdapterDescription = $connection.LocalAdapterDescription HostMacAddress = $connection.LocalMacAddress SwitchPortId = $connection.RemotePortId } $nodeKey = $connection.LocalHostName if ($switch2NodeDict.ContainsKey($switchKey)) { if ($switch2NodeDict[$switchKey].ContainsKey($nodeKey)) { $switch2NodeDict[$switchKey][$nodeKey] += $connectItem } else { $switch2NodeDict[$switchKey][$nodeKey] = @($connectItem) } } else { $switch2NodeDict[$switchKey] = @{ $nodeKey = @($connectItem) } } } $switch2NodeJsonFile = Join-Path -Path $OutputPath -ChildPath "Switch2Node.json" $switch2NodeDict | ConvertTo-Json -Depth 5 | Out-File -FilePath $switch2NodeJsonFile -Encoding utf8 Log-Info "Connection map saved to $switch2NodeJsonFile" $firstKey = $switch2NodeDict.Keys | Select-Object -First 1 $expectedLength = $switch2NodeDict[$firstKey].Count foreach ($switchKey in $switch2NodeDict.Keys) { $length = $switch2NodeDict[$switchKey].Count if ($length -ne $expectedLength) { $LLDPConnectionStatusMessage += "`n Switch '$switchKey' expects to be connected to $expectedLength hosts but is connected to $length." Log-Info $LLDPConnectionStatusMessage -Type 'WARNING' $LLDPConnectionRstObject = @{ Name = "Network_Switch_Connect_Different_Number_of_Nodes" Title = 'Switch Connected to a Different Number of Hosts' DisplayName = 'Switch Connected to a Different Number of Hosts' Severity = 'WARNING' Description = "Switch '$switchKey' expects to be connected to $expectedLength hosts but is connected to $length." Tags = @{ } Remediation = $lnTxt.ConnectionMismatchRemidation TargetResourceID = 'SwitchHostConnectionMismatch' TargetResourceName = 'SwitchHostConnectionMismatch' TargetResourceType = 'SwitchHostConnectionMismatch' Timestamp = [datetime]::UtcNow Status = 'FAILURE' AdditionalData = @{ Source = 'Connections.json' Resource = 'LLDPConnectionsValidated' Detail = $LLDPConnectionStatusMessage Status = 'FAILURE' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $LLDPConnectionResults += New-AzStackHciResultObject @LLDPConnectionRstObject return $LLDPConnectionResults } } # Passed all validation checks $LLDPConnectionStatusMessage += "`nPassed all validation checks." $LLDPConnectionRstObject = @{ Name = "AzStackHci_Hosts_LLDP_Connections_Validation" Title = 'LLDP Connections Validation Passed' DisplayName = 'LLDP Connections Validation Passed' Severity = 'INFO' Description = 'All LLDP connections between hosts and switches were successfully validated.' Tags = @{ } Remediation = '' TargetResourceID = 'LLDPConnectionsValidated' TargetResourceName = 'LLDPConnectionsValidated' TargetResourceType = 'LLDPConnectionsValidated' Timestamp = [datetime]::UtcNow Status = 'SUCCESS' AdditionalData = @{ Source = $LLDPJsonFile Resource = 'LLDPConnectionsValidated' Detail = $LLDPConnectionStatusMessage Status = 'SUCCESS' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $LLDPConnectionResults += New-AzStackHciResultObject @LLDPConnectionRstObject return $LLDPConnectionResults } catch { Log-Info "An error occurred while testing LLDP connections: $_" -Type ERROR } finally { # Log the completion of the test Log-Info "Completed Test-LLDPConnections" } } # Function to validate LLDP Storage VLAN configuration function Test-LLDPStorageVlan { param ( [string] $OutputPath, # Path to the output directory where the LLDP JSON file is located [hashtable]$StorageAdapterVLANIDInfo # Hashtable containing storage adapter names and their expected VLAN IDs ) # Construct the path for the merged LLDP JSON file $mergedLLDPJson = Join-Path -Path $OutputPath -ChildPath "MergedLLDPData.json" # Check if the JSON file exists if (-Not (Test-Path -Path $mergedLLDPJson)) { throw "Error: The file '$mergedLLDPJson' does not exist. Please verify the file path." } # Parse the JSON file to retrieve LLDP data $mergedLLDPJsonObj = Get-Content -Path $mergedLLDPJson | ConvertFrom-Json # Initialize the return object to store results $retVal = [PSCustomObject]@{ Pass = $true # Indicates if the validation passed or failed Message = "Validating Storage VLAN ID Configuration..." } # Iterate through each host in the JSON data foreach ($node in $mergedLLDPJsonObj.PSObject.Properties) { $nodeName = $node.Name # Host name $nodeObject = $node.Value.PSObject.Properties # Host properties # Loop through the storage adapters provided in the input hashtable foreach ($adapter in $StorageAdapterVLANIDInfo.GetEnumerator()) { $adapterName = $adapter.Key # Adapter name $expectedVLANID = $adapter.Value # Expected VLAN ID # Check if the storage adapter is present in the JSON data $adapterProperty = $nodeObject | Where-Object { $_.Name -eq $adapterName } if (-not $adapterProperty) { $retVal.Pass = $false $retVal.Message += "`n Warning: Storage adapter [$adapterName] could not be found on host [$nodeName]. Please check the adapter name and try again." continue } # Retrieve the adapter details and VLAN IDs $adapterObject = $adapterProperty.Value $adapterVLANIDs = $adapterObject.RemoteVLANIDs # Check if VLAN IDs are missing or unknown if ((-not $adapterVLANIDs) -or ($adapterVLANIDs.Count -eq 0) -or ($adapterVLANIDs -eq "Unknown")) { $retVal.Pass = $false $retVal.Message += "`n Warning: No VLAN ID detected on adapter [$adapterName] of host [$nodeName]. Please verify the adapter and try again." continue } # Check if the VLAN ID matches the expected value if ($adapterVLANIDs -notcontains $expectedVLANID) { $retVal.Pass = $false $retVal.Message += "`n Warning: Expected VLAN ID [ $expectedVLANID ], but found [ $adapterVLANIDs ] for adapter [$adapterName] on host [$nodeName]." continue } } } # Set the final status based on the validation results $TestStatus = if ($retVal.Pass) { 'SUCCESS' } else { 'FAILURE' } $TestDetailMsg = $retVal.Message # Create the result object to store the validation outcome $TestRstObject = @{ Name = 'AzStackHci_LLDP_Test_Storage_VLAN_Config' Title = 'Validate Storage VLAN Configuration' DisplayName = 'Validate Storage VLAN Configuration' Severity = 'WARNING' Description = 'This test compares the storage VLAN ID from LLDP TLVs against the input VLAN ID configuration.' Tags = @{} Remediation = $lnTxt.ValidateStorageVLANIDRemidation TargetResourceID = 'ValidateStorageVLANID' TargetResourceName = 'ValidateStorageVLANID' TargetResourceType = 'ValidateStorageVLANID' Timestamp = [datetime]::UtcNow Status = $TestStatus AdditionalData = @{ Source = 'MergedLLDPData.json' Resource = 'ValidateStorageVLANID' Detail = $TestDetailMsg Status = $TestStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } # Append the test result to the results array and return it $TestResults += New-AzStackHciResultObject @TestRstObject return $TestResults } # Function to validate LLDP DCBX (Data Center Bridging eXchange) Configuration: ETS and PFC function Test-LLDPDcbxConfiguration { param ( [string] $OutputPath, # Path to the output directory where the LLDP JSON file is located [hashtable]$StorageAdapterVLANIDInfo, # Hashtable containing storage adapter names and their expected VLAN IDs [hashtable]$DcbxConfigInfo # Hashtable containing the expected DCBX configuration for ETS and PFC ) # Construct the path for the merged LLDP JSON file $mergedLLDPJson = Join-Path -Path $OutputPath -ChildPath "MergedLLDPData.json" # Check if the JSON file exists if (-Not (Test-Path -Path $mergedLLDPJson)) { Log-Info "The file '$mergedLLDPJson' does not exist." -Type ERROR throw "Error: The file '$mergedLLDPJson' does not exist. Please verify the file path." } # Parse the JSON file to retrieve LLDP data $mergedLLDPJsonObj = Get-Content -Path $mergedLLDPJson | ConvertFrom-Json # Initialize the return object to store results $retVal = [PSCustomObject]@{ Pass = $true # Indicates if the validation passed or failed Message = "Validating DCBX Configuration..." } # Iterate through each host in the JSON data foreach ($node in $mergedLLDPJsonObj.PSObject.Properties) { $nodeName = $node.Name # Host name $nodeObject = $node.Value.PSObject.Properties # Host properties # Loop through the storage adapters provided in the input hashtable foreach ($adapter in $StorageAdapterVLANIDInfo.GetEnumerator()) { $adapterName = $adapter.Key # Adapter name # Check if the storage adapter is present in the JSON data $adapterProperty = $nodeObject | Where-Object { $_.Name -eq $adapterName } if (-not $adapterProperty) { $retVal.Pass = $false $retVal.Message += "`n Warning: Storage adapter [$adapterName] not found on host [$nodeName]. Please verify the adapter name and try again." continue } # Retrieve the adapter's ETS and PFC configurations $adapterObject = $adapterProperty.Value $nbrLldpEts = $adapterObject.RemoteETS $nbrLldpPfc = $adapterObject.RemotePFC # Validate if ETS configuration is missing or unknown if ((-not $nbrLldpEts) -or ($nbrLldpEts.Count -eq 0) -or ($nbrLldpEts -eq "Unknown")) { $retVal.Pass = $false $retVal.Message += "`n Warning: ETS configuration not detected for adapter [$adapterName] on host [$nodeName]." continue } # Validate if PFC configuration is missing or unknown if ((-not $nbrLldpPfc) -or ($nbrLldpPfc.Count -eq 0) -or ($nbrLldpPfc -eq "Unknown")) { $retVal.Pass = $false $retVal.Message += "`n Warning: PFC configuration not detected for adapter [$adapterName] on host [$nodeName]." continue } # Validate if ETS configuration matches the expected configuration $nbrLldpEts = $nbrLldpEts -join ',' $expectedLldpEts = $DcbxConfigInfo["ETS"] -join ',' if (($DcbxConfigInfo["ETS"].Count -ne $nbrLldpEts.Split(',').Count) -or (-not ($expectedLldpEts -eq $nbrLldpEts))) { $retVal.Pass = $false $retVal.Message += "`n Warning: Expected ETS configuration [$expectedLldpEts] but received [$nbrLldpEts] for adapter [$adapterName] on host [$nodeName]." continue } # Validate if PFC configuration matches the expected configuration $nbrLldpPfc = $nbrLldpPfc -join ',' $expectedLldpPfc = $DcbxConfigInfo["PFC"] -join ',' if (($DcbxConfigInfo["PFC"].Count -ne $nbrLldpPfc.Split(',').Count) -or (-not ($expectedLldpPfc -eq $nbrLldpPfc))) { $retVal.Pass = $false $retVal.Message += "`n Warning: Expected PFC configuration [$expectedLldpPfc] but received [$nbrLldpPfc] for adapter [$adapterName] on host [$nodeName]." continue } } } # Set the final status based on the validation results $TestStatus = if ($retVal.Pass) { 'SUCCESS' } else { 'FAILURE' } $TestDetailMsg = $retVal.Message # Create the result object to store the validation outcome $TestRstObject = @{ Name = 'AzStackHci_LLDP_Test_Neighbor_DCBX_Config' Title = 'Validate Neighbor LLDP DCBX Configuration' DisplayName = 'Validate Neighbor LLDP DCBX Configuration' Severity = 'WARNING' Description = 'This test compares the Neighbor LLDP DCBX configuration with the expected ETS and PFC settings.' Tags = @{} Remediation = $lnTxt.ValidateNeighborDCBXConfigRemidation TargetResourceID = 'ValidateNeighborDCBXConfig' TargetResourceName = 'ValidateNeighborDCBXConfig' TargetResourceType = 'ValidateNeighborDCBXConfig' Timestamp = [datetime]::UtcNow Status = $TestStatus AdditionalData = @{ Source = 'MergedLLDPData.json' Resource = 'ValidateNeighborDCBXConfig' Detail = $TestDetailMsg Status = $TestStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } # Append the test result to the results array and return it $TestResults += New-AzStackHciResultObject @TestRstObject return $TestResults } function Test-LLDPAvailabilityZoneConnections { <# .SYNOPSIS Validates if network switch configurations are valid based on Availability Zones .DESCRIPTION This function performs two main checks for deployments configured with a 'RackAware' cluster pattern and multiple Availability Zones: 1. Intra-Zone Consistency: Ensures all nodes within the same Availability Zone connect to the exact same set of switches. 2. Cross-Zone Isolation: Ensures that nodes in different Availability Zones connect to completely separate sets of switches (no overlap). .PARAMETER ClusterPattern The cluster pattern specified in the deployment configuration (e.g., 'RackAware'). Validation is primarily relevant for 'RackAware'. .PARAMETER LocalAvailabilityZones An array of objects representing the defined local availability zones, each containing a name and a list of nodes. .PARAMETER OutputPath Path to the output directory containing Node2Switch.json and NodeName2Ip.json generated by previous LLDP tests. .NOTES Requires Node2Switch.json and NodeName2Ip.json to exist in the OutputPath. Generates WARNING level for configuration mismatches, allowing deployment to proceed. #> [CmdletBinding()] param ( [System.String] $ClusterPattern, [array] $LocalAvailabilityZones, [System.String] $OutputPath ) $TestResults = @() $availabilityZoneResults = @{} try { $localZones = $LocalAvailabilityZones $clusterPattern = $ClusterPattern if (-not $PSBoundParameters.ContainsKey('ClusterPattern') -or -not $PSBoundParameters.ContainsKey('LocalAvailabilityZones')) { Log-Info "ClusterPattern or LocalAvailabilityZones parameters not provided. Skipping Availability Zone validation (likely not called with an answer file context)." return $TestResults } if ($null -eq $localZones -or $localZones.Count -eq 0) { Log-Info "No 'LocalAvailabilityZones' data provided. Skipping Availability Zone specific connection validation." return $TestResults } if ($null -eq $clusterPattern -or $clusterPattern -ne 'RackAware') { Log-Info "Cluster pattern is '$clusterPattern' (not 'RackAware'). Skipping Test" return $TestResults } if ($localZones.Count -lt 2) { Log-Info "Only one Availability Zone defined. Skipping cross-zone switch overlap check" } Log-Info "Starting Availability Zone Connection Validation." $node2SwitchJsonFile = Join-Path -Path $OutputPath -ChildPath "Node2Switch.json" $nodeName2IpJsonFile = Join-Path -Path $OutputPath -ChildPath "NodeName2Ip.json" if (-not (Test-Path -Path $node2SwitchJsonFile -PathType Leaf)) { $warnObj = @{ Name = 'AzStackHci_LLDP_Prerequisite_Missing_Node2Switch_File' Title = 'Prerequisite File Missing for Availability Zone Validation' DisplayName = 'Prerequisite File Missing for Availability Zone Validation' Severity = 'WARNING' Description = "Required file Node2Switch.json not found in '$OutputPath'. Cannot perform Availability Zone connection validation. Ensure 'Test-LLDPConnections' ran successfully." Tags = @{ ZoneValidation = 'Prerequisite' } Remediation = "Ensure 'Test-LLDPConnections' completes successfully before running Availability Zone checks. Verify the OutputPath '$OutputPath'." TargetResourceID = 'AvailabilityZoneConnectionValidation' TargetResourceName = 'Node2Switch.json' TargetResourceType = 'File' Timestamp = [datetime]::UtcNow Status = 'FAILURE' AdditionalData = @{ Source = $MyInvocation.MyCommand.Name Resource = 'AvailabilityZoneConnectionValidation' Detail = "Node2Switch.json is missing, which is required for this validation." Status = 'FAILURE' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $TestResults += New-AzStackHciResultObject @warnObj Log-Info "Required file Node2Switch.json not found in '$OutputPath'. Skipping Availability Zone validation." -Type Warning return $TestResults } if (-not (Test-Path -Path $nodeName2IpJsonFile -PathType Leaf)) { $warnObj = @{ Name = 'AzStackHci_LLDP_Prerequisite_Missing_NodeName2Ip_File' Title = 'Prerequisite File Missing for Availability Zone Validation' DisplayName = 'Prerequisite File Missing for Availability Zone Validation' Severity = 'WARNING' Description = "Required file NodeName2Ip.json not found in '$OutputPath'. Cannot perform Availability Zone connection validation. Ensure 'Test-LLDPConnections' ran successfully and had node name/IP info." Tags = @{ ZoneValidation = 'Prerequisite' } Remediation = "Ensure 'Test-LLDPConnections' completes successfully and generated NodeName2Ip.json. Verify the OutputPath '$OutputPath'." TargetResourceID = 'AvailabilityZoneConnectionValidation' TargetResourceName = 'NodeName2Ip.json' TargetResourceType = 'File' Timestamp = [datetime]::UtcNow Status = 'FAILURE' AdditionalData = @{ Source = $MyInvocation.MyCommand.Name Resource = 'AvailabilityZoneConnectionValidation' Detail = "NodeName2Ip.json is missing, which is required for this validation." Status = 'FAILURE' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $TestResults += New-AzStackHciResultObject @warnObj Log-Info "Required file NodeName2Ip.json not found in '$OutputPath'. Skipping Availability Zone validation." -Type Warning return $TestResults } Log-Info "Loading data from Node2Switch.json and NodeName2Ip.json." $node2SwitchData = Get-Content $node2SwitchJsonFile -Raw | ConvertFrom-Json $nodeName2IpObject = Get-Content $nodeName2IpJsonFile -Raw | ConvertFrom-Json $nodeName2IpMap = @{} if ($nodeName2IpObject) { foreach ($property in $nodeName2IpObject.PSObject.Properties) { $nodeName2IpMap[$property.Name] = $property.Value } Log-Info "Successfully loaded NodeName to IP Map." } else { Log-Info "NodeName2Ip.json file at '$nodeName2IpJsonFile' is empty or could not be parsed correctly." -Type Error return $TestResults } $zoneSwitchSets = @{} $intraZoneFailures = @{} $crossZoneFailures = @() $intraZoneStatus = 'SUCCESS' $crossZoneStatus = 'SUCCESS' #TEST CASE 1: Intra-Zone Switch Isolation Log-Info "Starting Test Case 1: Intra-Zone Switch Consistency Validation." foreach ($zone in $localZones) { $zoneName = $zone.localAvailabilityZoneName Log-Info "Validating Zone: '$zoneName'" $intraZoneFailures[$zoneName] = @() [string[]]$expectedSwitchSet = $null $firstNodeProcessed = $false if ($null -eq $zone.nodes -or $zone.nodes.Count -eq 0) { Log-Info "Zone '$zoneName' contains no nodes in the provided data. Skipping." -Type Warning $availabilityZoneResults[$zoneName] = [PSCustomObject]@{ ZoneName = $zoneName Status = 'SKIPPED' Messages = @("Zone definition contained no nodes.") ExpectedSwitchSet = "N/A" } continue } foreach ($nodeName in $zone.nodes) { # Find Node IP $nodeIp = $nodeName2IpMap[$nodeName] if (-not $nodeIp) { $msg = "Node '$nodeName' in Zone '$zoneName' not found in NodeName2Ip mapping ($nodeName2IpJsonFile). Cannot verify its switch connections." Log-Info "Node data lookup failed for a node in Zone '$zoneName'." -Type Warning $intraZoneFailures[$zoneName] += $msg $intraZoneStatus = 'FAILURE' if (-not $firstNodeProcessed) { $expectedSwitchSet = @("ERROR_NODE_IP_MISSING_FOR_$($nodeName)") } continue } # Find Switches for this Node IP if (-not $node2SwitchData.PSObject.Properties.Name.Contains($nodeIp)) { $msg = "Node '$nodeName' (IP: $nodeIp) in Zone '$zoneName' not found in Node2Switch data ($node2SwitchJsonFile). LLDP data might be missing or incomplete for this node." Log-Info "LLDP data lookup failed for a node in Zone '$zoneName'." -Type Warning $intraZoneFailures[$zoneName] += $msg $intraZoneStatus = 'FAILURE' if (-not $firstNodeProcessed) { $expectedSwitchSet = @("ERROR_NODE_LLDP_DATA_MISSING_FOR_$($nodeName)") } continue } # Get the switch identifiers (keys) for the current node's IP $nodeSwitchInfo = $node2SwitchData.$($nodeIp) if ($null -eq $nodeSwitchInfo) { $msg = "Node '$nodeName' (IP: $nodeIp) in Zone '$zoneName' has null switch data in Node2Switch data ($node2SwitchJsonFile). Unexpected format." Log-Info "Node in Zone '$zoneName' has unexpected null switch data." $intraZoneFailures[$zoneName] += $msg $intraZoneStatus = 'FAILURE' if (-not $firstNodeProcessed) { $expectedSwitchSet = @("ERROR_NULL_SWITCH_DATA_FOR_$($nodeName)") } continue } $currentNodeSwitches = ($nodeSwitchInfo.PSObject.Properties | Select-Object -ExpandProperty Name) | Sort-Object if (-not $firstNodeProcessed) { $expectedSwitchSet = $currentNodeSwitches $zoneSwitchSets[$zoneName] = $expectedSwitchSet $firstNodeProcessed = $true Log-Info "Zone '$zoneName': expected switch set established." } else { if ($expectedSwitchSet -match "^ERROR_") { Log-Info "Zone '$zoneName': Cannot compare node as the expected switch set could not be established due to previous errors." -Type Warning continue } $comparison = Compare-Object -ReferenceObject $expectedSwitchSet -DifferenceObject $currentNodeSwitches -SyncWindow 0 if ($comparison) { $intraZoneStatus = 'FAILURE' $missingSwitches = ($comparison | Where-Object SideIndicator -eq '<=').InputObject -join ', ' $extraSwitches = ($comparison | Where-Object SideIndicator -eq '=>').InputObject -join ', ' $msg = "Node '$nodeName' (IP: $nodeIp) in Zone '$zoneName' has inconsistent switch connections." if ($missingSwitches) { $msg += " Expected switches not found: [$missingSwitches]." } if ($extraSwitches) { $msg += " Unexpected switches found: [$extraSwitches]." } $msg += " Expected set from first node: [$($expectedSwitchSet -join ', ')]" Log-Info "Node in Zone '$zoneName' has inconsistent switch connections. Ensure all nodes in the same zone are connected to same set of switches." -Type Warning $intraZoneFailures[$zoneName] += $msg } } } # Store results for this zone $zoneStatus = 'FAILURE' if (-not $firstNodeProcessed -and $intraZoneFailures[$zoneName].Count -eq 0) { $zoneStatus = 'FAILURE' if($null -eq $expectedSwitchSet) {$expectedSwitchSet = @("ERROR_NO_VALID_NODES_IN_ZONE")} } elseif ($intraZoneFailures[$zoneName].Count -eq 0 -and $firstNodeProcessed) { $zoneStatus = 'SUCCESS' } $availabilityZoneResults[$zoneName] = [PSCustomObject]@{ ZoneName = $zoneName Status = $zoneStatus Messages = $intraZoneFailures[$zoneName] ExpectedSwitchSet = if($expectedSwitchSet -is [array]){$expectedSwitchSet -join ', '} else {$expectedSwitchSet} } } #TEST CASE 2: Cross-Zone Switch Isolation Log-Info "Starting Test Case 2: Cross-Zone Switch Isolation Validation." # Check if we have at least two zones with successfully determined switch sets $validZoneSwitchSets = $zoneSwitchSets.GetEnumerator() | Where-Object { $_.Value -ne $null -and $_.Value -notmatch "^ERROR_" } | Select-Object -Property Key, Value if ($validZoneSwitchSets.Count -ge 2) { $zoneNames = $validZoneSwitchSets.Key | Sort-Object for ($i = 0; $i -lt ($zoneNames.Count - 1); $i++) { for ($j = $i + 1; $j -lt $zoneNames.Count; $j++) { $zoneA = $zoneNames[$i] $zoneB = $zoneNames[$j] $switchesA = $zoneSwitchSets[$zoneA] $switchesB = $zoneSwitchSets[$zoneB] # Find common switches (intersection) if (($switchesA -is [array]) -and ($switchesB -is [array])) { $overlap = Compare-Object -ReferenceObject $switchesA -DifferenceObject $switchesB -IncludeEqual -ExcludeDifferent -SyncWindow 0 | Select-Object -ExpandProperty InputObject if ($overlap) { $crossZoneStatus = 'FAILURE' $msg = "Switch overlap detected between Zone '$zoneA' (Switches: [$($switchesA -join ', ')]) and Zone '$zoneB' (Switches: [$($switchesB -join ', ')]). Shared switches: [$($overlap -join ', ')]. Zones should use distinct sets of switches in a RackAware clusyersn." Log-Info "Switch overlap detected between Zone '$zoneA' and Zone '$zoneB'. This isn't recommended for RAC." -Type Warning $crossZoneFailures += $msg } else { Log-Info "No switch overlap found between Zone '$zoneA' and Zone '$zoneB'." } } else { Log-Info "Skipping overlap check between Zone '$zoneA' and Zone '$zoneB' due to invalid switch data format (expected array)." -Type Warning } } } } else { Log-Info "Skipping cross-zone switch overlap check as fewer than two zones have complete and valid switch data." } #Generate Final Result Objects $intraZoneDetailMsg = "Intra-Zone Switch Consistency Check Results:`n" if ($availabilityZoneResults.Count -gt 0) { $intraZoneDetailMsg += ($availabilityZoneResults.Values | Format-Table -AutoSize | Out-String) } else { $intraZoneDetailMsg += "No zones processed or found." } $crossZoneDetailMsg = "Cross-Zone Switch Isolation Check Results:`n" if ($crossZoneFailures.Count -gt 0) { $crossZoneDetailMsg += ($crossZoneFailures -join "`n") } elseif ($localZones.Count -ge 2 -and $validZoneSwitchSets.Count -ge 2) { # Only report success if the check was actually performed $crossZoneDetailMsg += "No switch overlaps detected between different zones with valid data." } elseif ($localZones.Count -ge 2) { # Check was applicable but skipped due to lack of valid data $crossZoneDetailMsg += "Cross-zone Check skipped due to insufficient valid switch data from zones." } else { # Check was not applicable (fewer than 2 zones) $crossZoneDetailMsg += "Cross-zone Check not applicable (fewer than 2 zones defined)." } # Result for Intra-Zone Check $intraZoneResultObj = @{ Name = 'AzStackHci_LLDP_Test_Intra_Zone_Switch_Consistency' Title = 'Validate Intra-Zone Switch Consistency (RackAware)' DisplayName = 'Validate Intra-Zone Switch Consistency' Severity = 'WARNING' Description = "Checks if all nodes within the same Availability Zone connect to the exact same set of switches, as expected in a RackAware pattern. Requires Node2Switch.json and NodeName2Ip.json." Tags = @{ ZoneValidation = 'IntraZone' } Remediation = "Ensure all nodes within a single zone (e.g., rack) are cabled identically to the same set of ToR switches. Verify LLDP is enabled and functioning correctly on all relevant host ports and switch ports. Check Node2Switch.json ($node2SwitchJsonFile) and NodeName2Ip.json ($nodeName2IpJsonFile) for details. Review detailed results." # Potentially use $lnTxt.IntraZoneRemediation TargetResourceID = 'AllZones' TargetResourceName = 'IntraZoneSwitchConsistency' TargetResourceType = 'AvailabilityZoneNetworkPolicy' Timestamp = [datetime]::UtcNow Status = $intraZoneStatus AdditionalData = @{ Source = $node2SwitchJsonFile Resource = 'IntraZoneSwitchConsistency' Detail = $intraZoneDetailMsg Status = $intraZoneStatus TimeStamp = [datetime]::UtcNow ZoneResults = ($availabilityZoneResults | ConvertTo-Json -Depth 10 -Compress) } HealthCheckSource = $ENV:EnvChkrId } $TestResults += New-AzStackHciResultObject @intraZoneResultObj # Result for Cross-Zone Check if ($localZones.Count -ge 2) { $crossZoneResultObj = @{ Name = 'AzStackHci_LLDP_Test_Cross_Zone_Switch_Isolation' Title = 'Validate Cross-Zone Switch Isolation (RackAware)' DisplayName = 'Validate Cross-Zone Switch Isolation' Severity = 'WARNING' Description = "Checks if different Availability Zones connect to completely distinct sets of switches, ensuring network isolation between zones (racks) as expected in a RackAware pattern. Requires Node2Switch.json." Tags = @{ ZoneValidation = 'CrossZone' } Remediation = "Ensure that nodes in different zones (e.g., racks) are connected to separate sets of ToR switches. There should be no shared switches between zones. Verify cabling and LLDP data. Check Node2Switch.json ($node2SwitchJsonFile) for details. Review detailed results." TargetResourceID = 'AllZones' TargetResourceName = 'CrossZoneSwitchIsolation' TargetResourceType = 'AvailabilityZoneNetworkPolicy' Timestamp = [datetime]::UtcNow Status = $crossZoneStatus AdditionalData = @{ Source = $node2SwitchJsonFile Resource = 'CrossZoneSwitchIsolation' Detail = $crossZoneDetailMsg Status = $crossZoneStatus TimeStamp = [datetime]::UtcNow ZoneSwitchSets = ($zoneSwitchSets | ConvertTo-Json -Depth 10 -Compress) } HealthCheckSource = $ENV:EnvChkrId } $TestResults += New-AzStackHciResultObject @crossZoneResultObj } Log-Info "Finished Availability Zone Connection Validation." return $TestResults } catch { $errMsg = "An error occurred during Availability Zone Connection Validation: $($_.Exception.Message) - $($_.ScriptStackTrace)" Log-Info $errMsg -Type Error # Create a generic error result for the function failure $errorObj = @{ Name = 'AzStackHci_LLDP_Test_Availability_Zone_Connection_Error' Title = 'Availability Zone Connection Validation Failed' DisplayName = 'Availability Zone Connection Validation Failed' Severity = 'CRITICAL' Description = "An unexpected error occurred while performing the Availability Zone connection validation." Tags = @{ ZoneValidation = 'Error' } Remediation = "Review the error details and logs in '$OutputPath'. Check prerequisites like file existence and content format. Error: $($_.Exception.Message)" TargetResourceID = 'AvailabilityZoneConnectionValidation' TargetResourceName = 'AvailabilityZoneConnectionValidation' TargetResourceType = 'AvailabilityZoneNetworkPolicy' Timestamp = [datetime]::UtcNow Status = 'FAILURE' AdditionalData = @{ Source = $MyInvocation.MyCommand.Name Resource = 'AvailabilityZoneConnectionValidation' Detail = $errMsg Status = 'FAILURE' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $TestResults += New-AzStackHciResultObject @errorObj Log-Info "Finished Availability Zone Connection Validation." return $TestResults } } Export-ModuleMember -Function Test-LLDPNbrTlvs, Test-MergedLLDPDataToJson, Test-LLDPConnections, Test-LLDPStorageVlan, Test-LLDPDcbxConfiguration, Test-LLDPAvailabilityZoneConnections, Export-MergedLLDPDataToJson # SIG # Begin signature block # MIIoUgYJKoZIhvcNAQcCoIIoQzCCKD8CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAYiABoSqDwnZ58 # SfthFLJjO3FgPltIr0vqOWEsQndwqqCCDYUwggYDMIID66ADAgECAhMzAAAEA73V # lV0POxitAAAAAAQDMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjQwOTEyMjAxMTEzWhcNMjUwOTExMjAxMTEzWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQCfdGddwIOnbRYUyg03O3iz19XXZPmuhEmW/5uyEN+8mgxl+HJGeLGBR8YButGV # LVK38RxcVcPYyFGQXcKcxgih4w4y4zJi3GvawLYHlsNExQwz+v0jgY/aejBS2EJY # oUhLVE+UzRihV8ooxoftsmKLb2xb7BoFS6UAo3Zz4afnOdqI7FGoi7g4vx/0MIdi # kwTn5N56TdIv3mwfkZCFmrsKpN0zR8HD8WYsvH3xKkG7u/xdqmhPPqMmnI2jOFw/ # /n2aL8W7i1Pasja8PnRXH/QaVH0M1nanL+LI9TsMb/enWfXOW65Gne5cqMN9Uofv # ENtdwwEmJ3bZrcI9u4LZAkujAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQU6m4qAkpz4641iK2irF8eWsSBcBkw # VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzUwMjkyNjAfBgNVHSMEGDAW # gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw # MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx # XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB # AFFo/6E4LX51IqFuoKvUsi80QytGI5ASQ9zsPpBa0z78hutiJd6w154JkcIx/f7r # EBK4NhD4DIFNfRiVdI7EacEs7OAS6QHF7Nt+eFRNOTtgHb9PExRy4EI/jnMwzQJV # NokTxu2WgHr/fBsWs6G9AcIgvHjWNN3qRSrhsgEdqHc0bRDUf8UILAdEZOMBvKLC # rmf+kJPEvPldgK7hFO/L9kmcVe67BnKejDKO73Sa56AJOhM7CkeATrJFxO9GLXos # oKvrwBvynxAg18W+pagTAkJefzneuWSmniTurPCUE2JnvW7DalvONDOtG01sIVAB # +ahO2wcUPa2Zm9AiDVBWTMz9XUoKMcvngi2oqbsDLhbK+pYrRUgRpNt0y1sxZsXO # raGRF8lM2cWvtEkV5UL+TQM1ppv5unDHkW8JS+QnfPbB8dZVRyRmMQ4aY/tx5x5+ # sX6semJ//FbiclSMxSI+zINu1jYerdUwuCi+P6p7SmQmClhDM+6Q+btE2FtpsU0W # +r6RdYFf/P+nK6j2otl9Nvr3tWLu+WXmz8MGM+18ynJ+lYbSmFWcAj7SYziAfT0s # IwlQRFkyC71tsIZUhBHtxPliGUu362lIO0Lpe0DOrg8lspnEWOkHnCT5JEnWCbzu # iVt8RX1IV07uIveNZuOBWLVCzWJjEGa+HhaEtavjy6i7MIIHejCCBWKgAwIBAgIK # YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm # aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw # OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD # VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG # 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la # UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc # 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D # dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ # lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk # kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 # A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd # X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL # 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd # sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 # T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS # 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI # bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL # BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD # uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF # BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h # cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA # YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn # 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 # v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b # pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ # KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy # CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp # mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi # hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb # BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS # oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL # gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX # cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGiMwghofAgEBMIGVMH4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p # Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAAQDvdWVXQ87GK0AAAAA # BAMwDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIF/3 # GEYeoE5RuK7pzHCTRc/mvzyT5+cENKDXbrfXrsHsMEIGCisGAQQBgjcCAQwxNDAy # oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20wDQYJKoZIhvcNAQEBBQAEggEATkbCW6Y7Ms3mfXqGHn+as/bkVhQvMNuddUzd # YmaPCz6B6g6GQCdG/zbN/bTP/dWw7Ikhjnx1YjPK3vWT70gvlU9FgOzVgZm6g2Sk # 6eEHf0rn29K8PV0lDBJoPA13FPLOqfqGHEDaUXrMh6s9FGk5ZebbQUMgxPanylIl # odlgK37oM+8aSPM6PWgZPBa/v9sMWMZ+UeJIED4NkBCcFujgEPubtc8wTC4fQXU/ # QLyURWO0f84Txr3oPPbDxy82H41WQoUYfQzttn1t2j4P6FnlRwBi6nJYS2g+8xIT # CTJw8pd8rkQ9zHG+Epfu8zm/nrndlgRj9K0S0Q2ZYKopwPu7naGCF60wghepBgor # BgEEAYI3AwMBMYIXmTCCF5UGCSqGSIb3DQEHAqCCF4YwgheCAgEDMQ8wDQYJYIZI # AWUDBAIBBQAwggFaBgsqhkiG9w0BCRABBKCCAUkEggFFMIIBQQIBAQYKKwYBBAGE # WQoDATAxMA0GCWCGSAFlAwQCAQUABCBSrXYClMlKvxMNg8/F4y2lPjU16wv1lh0g # DRJtWPL+GgIGaC3ymZVwGBMyMDI1MDYxMDE1NDgzNC4xNDFaMASAAgH0oIHZpIHW # MIHTMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQL # EyRNaWNyb3NvZnQgSXJlbGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsT # Hm5TaGllbGQgVFNTIEVTTjoyQTFBLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9z # b2Z0IFRpbWUtU3RhbXAgU2VydmljZaCCEfswggcoMIIFEKADAgECAhMzAAAB+R9n # jXWrpPGxAAEAAAH5MA0GCSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w # IFBDQSAyMDEwMB4XDTI0MDcyNTE4MzEwOVoXDTI1MTAyMjE4MzEwOVowgdMxCzAJ # BgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25k # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1pY3Jv # c29mdCBJcmVsYW5kIE9wZXJhdGlvbnMgTGltaXRlZDEnMCUGA1UECxMeblNoaWVs # ZCBUU1MgRVNOOjJBMUEtMDVFMC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGlt # ZS1TdGFtcCBTZXJ2aWNlMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA # tD1MH3yAHWHNVslC+CBTj/Mpd55LDPtQrhN7WeqFhReC9xKXSjobW1ZHzHU8V2BO # JUiYg7fDJ2AxGVGyovUtgGZg2+GauFKk3ZjjsLSsqehYIsUQrgX+r/VATaW8/ONW # y6lOyGZwZpxfV2EX4qAh6mb2hadAuvdbRl1QK1tfBlR3fdeCBQG+ybz9JFZ45LN2 # ps8Nc1xr41N8Qi3KVJLYX0ibEbAkksR4bbszCzvY+vdSrjWyKAjR6YgYhaBaDxE2 # KDJ2sQRFFF/egCxKgogdF3VIJoCE/Wuy9MuEgypea1Hei7lFGvdLQZH5Jo2QR5uN # 8hiMc8Z47RRJuIWCOeyIJ1YnRiiibpUZ72+wpv8LTov0yH6C5HR/D8+AT4vqtP57 # ITXsD9DPOob8tjtsefPcQJebUNiqyfyTL5j5/J+2d+GPCcXEYoeWZ+nrsZSfrd5D # HM4ovCmD3lifgYnzjOry4ghQT/cvmdHwFr6yJGphW/HG8GQd+cB4w7wGpOhHVJby # 44kGVK8MzY9s32Dy1THnJg8p7y1sEGz/A1y84Zt6gIsITYaccHhBKp4cOVNrfoRV # Ux2G/0Tr7Dk3fpCU8u+5olqPPwKgZs57jl+lOrRVsX1AYEmAnyCyGrqRAzpGXyk1 # HvNIBpSNNuTBQk7FBvu+Ypi6A7S2V2Tj6lzYWVBvuGECAwEAAaOCAUkwggFFMB0G # A1UdDgQWBBSJ7aO6nJXJI9eijzS5QkR2RlngADAfBgNVHSMEGDAWgBSfpxVdAF5i # XYP05dJlpxtTNRnpcjBfBgNVHR8EWDBWMFSgUqBQhk5odHRwOi8vd3d3Lm1pY3Jv # c29mdC5jb20vcGtpb3BzL2NybC9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENB # JTIwMjAxMCgxKS5jcmwwbAYIKwYBBQUHAQEEYDBeMFwGCCsGAQUFBzAChlBodHRw # Oi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY3Jvc29mdCUyMFRp # bWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNydDAMBgNVHRMBAf8EAjAAMBYGA1Ud # JQEB/wQMMAoGCCsGAQUFBwMIMA4GA1UdDwEB/wQEAwIHgDANBgkqhkiG9w0BAQsF # AAOCAgEAZiAJgFbkf7jfhx/mmZlnGZrpae+HGpxWxs8I79vUb8GQou50M1ns7iwG # 2CcdoXaq7VgpVkNf1uvIhrGYpKCBXQ+SaJ2O0BvwuJR7UsgTaKN0j/yf3fpHD0kt # H+EkEuGXs9DBLyt71iutVkwow9iQmSk4oIK8S8ArNGpSOzeuu9TdJjBjsasmuJ+2 # q5TjmrgEKyPe3TApAio8cdw/b1cBAmjtI7tpNYV5PyRI3K1NhuDgfEj5kynGF/ui # zP1NuHSxF/V1ks/2tCEoriicM4k1PJTTA0TCjNbkpmBcsAMlxTzBnWsqnBCt9d+U # d9Va3Iw9Bs4ccrkgBjLtg3vYGYar615ofYtU+dup+LuU0d2wBDEG1nhSWHaO+u2y # 6Si3AaNINt/pOMKU6l4AW0uDWUH39OHH3EqFHtTssZXaDOjtyRgbqMGmkf8KI3qI # VBZJ2XQpnhEuRbh+AgpmRn/a410Dk7VtPg2uC422WLC8H8IVk/FeoiSS4vFodhnc # FetJ0ZK36wxAa3FiPgBebRWyVtZ763qDDzxDb0mB6HL9HEfTbN+4oHCkZa1HKl8B # 0s8RiFBMf/W7+O7EPZ+wMH8wdkjZ7SbsddtdRgRARqR8IFPWurQ+sn7ftEifaojz # uCEahSAcq86yjwQeTPN9YG9b34RTurnkpD+wPGTB1WccMpsLlM0wggdxMIIFWaAD # AgECAhMzAAAAFcXna54Cm0mZAAAAAAAVMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYD # VQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEe # MBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3Nv # ZnQgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0yMTA5MzAxODIy # MjVaFw0zMDA5MzAxODMyMjVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNo # aW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29y # cG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEw # MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA5OGmTOe0ciELeaLL1yR5 # vQ7VgtP97pwHB9KpbE51yMo1V/YBf2xK4OK9uT4XYDP/XE/HZveVU3Fa4n5KWv64 # NmeFRiMMtY0Tz3cywBAY6GB9alKDRLemjkZrBxTzxXb1hlDcwUTIcVxRMTegCjhu # je3XD9gmU3w5YQJ6xKr9cmmvHaus9ja+NSZk2pg7uhp7M62AW36MEBydUv626GIl # 3GoPz130/o5Tz9bshVZN7928jaTjkY+yOSxRnOlwaQ3KNi1wjjHINSi947SHJMPg # yY9+tVSP3PoFVZhtaDuaRr3tpK56KTesy+uDRedGbsoy1cCGMFxPLOJiss254o2I # 5JasAUq7vnGpF1tnYN74kpEeHT39IM9zfUGaRnXNxF803RKJ1v2lIH1+/NmeRd+2 # ci/bfV+AutuqfjbsNkz2K26oElHovwUDo9Fzpk03dJQcNIIP8BDyt0cY7afomXw/ # TNuvXsLz1dhzPUNOwTM5TI4CvEJoLhDqhFFG4tG9ahhaYQFzymeiXtcodgLiMxhy # 16cg8ML6EgrXY28MyTZki1ugpoMhXV8wdJGUlNi5UPkLiWHzNgY1GIRH29wb0f2y # 1BzFa/ZcUlFdEtsluq9QBXpsxREdcu+N+VLEhReTwDwV2xo3xwgVGD94q0W29R6H # XtqPnhZyacaue7e3PmriLq0CAwEAAaOCAd0wggHZMBIGCSsGAQQBgjcVAQQFAgMB # AAEwIwYJKwYBBAGCNxUCBBYEFCqnUv5kxJq+gpE8RjUpzxD/LwTuMB0GA1UdDgQW # BBSfpxVdAF5iXYP05dJlpxtTNRnpcjBcBgNVHSAEVTBTMFEGDCsGAQQBgjdMg30B # ATBBMD8GCCsGAQUFBwIBFjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3Bz # L0RvY3MvUmVwb3NpdG9yeS5odG0wEwYDVR0lBAwwCgYIKwYBBQUHAwgwGQYJKwYB # BAGCNxQCBAweCgBTAHUAYgBDAEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMB # Af8wHwYDVR0jBBgwFoAU1fZWy4/oolxiaNE9lJBb186aGMQwVgYDVR0fBE8wTTBL # oEmgR4ZFaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMv # TWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggr # BgEFBQcwAoY+aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNS # b29DZXJBdXRfMjAxMC0wNi0yMy5jcnQwDQYJKoZIhvcNAQELBQADggIBAJ1Vffwq # reEsH2cBMSRb4Z5yS/ypb+pcFLY+TkdkeLEGk5c9MTO1OdfCcTY/2mRsfNB1OW27 # DzHkwo/7bNGhlBgi7ulmZzpTTd2YurYeeNg2LpypglYAA7AFvonoaeC6Ce5732pv # vinLbtg/SHUB2RjebYIM9W0jVOR4U3UkV7ndn/OOPcbzaN9l9qRWqveVtihVJ9Ak # vUCgvxm2EhIRXT0n4ECWOKz3+SmJw7wXsFSFQrP8DJ6LGYnn8AtqgcKBGUIZUnWK # NsIdw2FzLixre24/LAl4FOmRsqlb30mjdAy87JGA0j3mSj5mO0+7hvoyGtmW9I/2 # kQH2zsZ0/fZMcm8Qq3UwxTSwethQ/gpY3UA8x1RtnWN0SCyxTkctwRQEcb9k+SS+ # c23Kjgm9swFXSVRk2XPXfx5bRAGOWhmRaw2fpCjcZxkoJLo4S5pu+yFUa2pFEUep # 8beuyOiJXk+d0tBMdrVXVAmxaQFEfnyhYWxz/gq77EFmPWn9y8FBSX5+k77L+Dvk # txW/tM4+pTFRhLy/AsGConsXHRWJjXD+57XQKBqJC4822rpM+Zv/Cuk0+CQ1Zyvg # DbjmjJnW4SLq8CdCPSWU5nR0W2rRnj7tfqAxM328y+l7vzhwRNGQ8cirOoo6CGJ/ # 2XBjU02N7oJtpQUQwXEGahC0HVUzWLOhcGbyoYIDVjCCAj4CAQEwggEBoYHZpIHW # MIHTMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQL # EyRNaWNyb3NvZnQgSXJlbGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsT # Hm5TaGllbGQgVFNTIEVTTjoyQTFBLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9z # b2Z0IFRpbWUtU3RhbXAgU2VydmljZaIjCgEBMAcGBSsOAwIaAxUAqs5WjWO7zVAK # mIcdwhqgZvyp6UaggYMwgYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz # aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv # cnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAx # MDANBgkqhkiG9w0BAQsFAAIFAOvyzTkwIhgPMjAyNTA2MTAxNTI2NDlaGA8yMDI1 # MDYxMTE1MjY0OVowdDA6BgorBgEEAYRZCgQBMSwwKjAKAgUA6/LNOQIBADAHAgEA # AgIP9DAHAgEAAgISaTAKAgUA6/QeuQIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgor # BgEEAYRZCgMCoAowCAIBAAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBCwUA # A4IBAQCr1I1ZQe9IsnHb/juf+dCBM6ChJLJ3EK9UsFWI3KCixH+4IW8QxayHQI3A # SAsChrMZ7sSIjNKnRdItwBj/FDhQJ18/LI1BBRUmxZ8wgpU81RVK+PcpLuz4TUVO # MZOtPDG0c3YhbUrQldXZgeftpTzJRnG0PCVRIGh3Fd3EK0PfmFvzsK5hwLBxOiDA # yF+WGgm+6LoV5Yz/ofLhwYkeCYVnB+plc+eZfutuQhhpKguJ06SNR0o9xUvURDEo # TLjqaNDef5JzrkkQXXJ9578E//CMYrjJzy7XGQhH9qDLLrvZEjJjeuK8+YBN7PUF # iECTcPUy6tPzLvdi3Bdx7vTzm+lkMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMC # VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV # BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp # bWUtU3RhbXAgUENBIDIwMTACEzMAAAH5H2eNdauk8bEAAQAAAfkwDQYJYIZIAWUD # BAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0B # CQQxIgQgHYr5n9uUD+HiL4gQwi8NfpAqwzm47D779KfuEd5Mpn8wgfoGCyqGSIb3 # DQEJEAIvMYHqMIHnMIHkMIG9BCA5I4zIHvCN+2T66RUOLCZrUEVdoKlKl8VeCO5S # bGLYEDCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u # MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp # b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB # +R9njXWrpPGxAAEAAAH5MCIEIJuMYGK4IyJaDhuuEYyg/mZmpAltD6Gq7O7JfgRU # i+f/MA0GCSqGSIb3DQEBCwUABIICABLOmawtKFb73MUCGSFKqUWR4Zn3JBXERKFB # U/zE//FQfVjBmWebgnudopX5dTT3TIwHx5jSw813uwMO5xdH3dILX2Kv8g1nEY9+ # bcbMtdptsIUY8n6Jtn5g9ZFl+mj0g7t/n5GNW4VuUwrDBKYom30Cu2R4P8NhQL65 # vtaIH6yL7IZOb3aT/EJ/K9h3lcmwsQQeKwhzjrhXnU/IuqGHxBMwWjr43PUeRUq5 # 1nMFyyerhS0GN3AWs+H/KDZRiKmcXdx8Tvz0KYvHipg0NqE/UDAR7JE+AO9na964 # nSKpWVqfhakISZ0GtDIhq6bTZHroLmM+Gw3txnqagwIqXN0clQ3xMoQJV31iCBNI # 7klHeuig/GBEqzT+ka4wxp5QqWhPeFpefPMXLPAnamAulwZ2lSUhILHhjVNvaq83 # IMuFezLZr7Ld+3K+TQvGPivWMzc35ENHtVa3W8D3vm9pGjR4/lH4IJMaZpUHBF3J # plgTBsVAK173Ega5nlD/Fcp3X2uA5fqMimb9n7SPkp6I5ofsHlUuS0mlBcsGKh48 # 8UvSJH5PR7oKCoxWEbprKnU9U9ILYE8Axs7bRoZY0tfwt/Q4yBp2Gru5QgZnoiZE # K00CZ0qCnkfuf2bv8KXAMQtZQF6v84h9szuFjaY8fKzfiBZ7ZD719bQIwej8JcFT # c9zxPhD+ # SIG # End signature block |