NetworkValidation/Microsoft.AzureStack.ReadinessChecker.NetworkValidation.psm1
|
<#############################################################
# # # Copyright (C) Microsoft Corporation. All rights reserved. # # # #############################################################> $global:ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $global:ProgressPreference = [System.Management.Automation.ActionPreference]::SilentlyContinue $moduleName = 'Microsoft.AzureStack.ReadinessChecker' Import-Module -Name $PSScriptRoot\..\Microsoft.AzureStack.ReadinessChecker.Reporting.psm1 -Force Import-Module -Name $PSScriptRoot\..\Microsoft.AzureStack.ReadinessChecker.Utilities.psm1 -Force <# .SYNOPSIS Verifies network infrastructure readiness for Azure Stack deployment. .DESCRIPTION Performs series of network tests from a device connected to the border switches to verify that network configuration prerequisites are met. .EXAMPLE Invoke-AzsNetworkValidation -SkipTests 'DnsServer' -TimeServer pool.ntp.org This example runs Azure Stack Edge network validation, skips the DNS Server test, and saves the report in the default location .EXAMPLE Invoke-AzsNetworkValidation -DeploymentDataPath D:\AzureStack\DeploymentData.json -CleanReport -OutputPath C:\Temp\ This example runs Azure Stack Hub network validation, creates a new report, and saves it to C:\Temp folder .NOTES This feature is in preview and may not produce the expected results. #> function Invoke-AzsNetworkValidation { [CmdletBinding(DefaultParameterSetName = 'Edge')] param ( # Path to Azure Stack Hub deployment configuration file created by the Deployment Worksheet. [Parameter(Mandatory = $true, ParameterSetName = 'Hub')] [Parameter(Mandatory = $true, ParameterSetName = 'HLH')] [System.String] $DeploymentDataPath, # List of test to run. Default is to run all tests. [Parameter(ParameterSetName = 'Edge')] [Parameter(ParameterSetName = 'Hub')] [ValidateSet( 'LinkLayer', 'PortChannel', 'BorderUplink', 'IPConfig', 'BgpPeering', 'BgpDefaultRoute', 'DnsServer', 'PathMtu', 'TimeServer', 'SyslogServer', 'Proxy', 'AzureEndpoint', 'AdfsEndpoint', 'Graph', 'CustomUrl', 'WindowsUpdateServer', 'DuplicateIP', 'DnsRegistration', 'DnsDelegation', 'HLH' )] [System.String[]] $RunTests, # List of test to skip. Default is to run all tests. [Parameter(ParameterSetName = 'Edge')] [Parameter(ParameterSetName = 'Hub')] [ValidateSet( 'PortChannel', 'BorderUplink', 'IPConfig', 'BgpPeering', 'BgpDefaultRoute', 'DnsServer', 'PathMtu', 'TimeServer', 'SyslogServer', 'AzureEndpoint', 'AdfsEndpoint', 'Graph', 'WindowsUpdateServer', 'DuplicateIP', 'DnsRegistration', 'DnsDelegation' )] [System.String[]] $SkipTests, # Path to the virtual router image. [Parameter(ParameterSetName = 'Hub')] [Parameter(ParameterSetName = 'HLH')] [System.String] $VirtualRouterImagePath, # DNS Server address(es) [Parameter(ParameterSetName = 'Edge')] [System.String[]] $DnsServer, # DNS name to resolve for DNS test [Parameter(ParameterSetName = 'Edge')] [Parameter(ParameterSetName = 'Hub')] [System.String] $DnsName = 'management.azure.com', # DNS name or IP address for the network path MTU test [Parameter(ParameterSetName = 'Edge')] [Parameter(ParameterSetName = 'Hub')] [System.String] $MtuTestDestination = 'go.microsoft.com', # Fully qualified domain name of the Edge device [Parameter(ParameterSetName = 'Edge')] [System.String] $DeviceFqdn, # Time Server address(es) [Parameter(ParameterSetName = 'Edge')] [System.String[]] $TimeServer, # Compute IP range(s) to be used by Kubernetes specified as Start IP and End IP separated by a hyphen # Example: '10.128.44.241-10.128.44.245' [Parameter(ParameterSetName = 'Edge')] [System.String[]] $ComputeIPs, # Azure cloud - specify to use a sovereign or custom Azure cloud [Parameter(ParameterSetName = 'Edge')] [ValidateSet('AzureCloud', 'AzureChinaCloud', 'AzureGermanCloud', 'AzureUSGovernment', 'AzureUSSec', 'AzureUSNat', 'CustomCloud')] [System.String] $AzureEnvironment = 'AzureCloud', # Azure Resource Manager endpoint URI for custom cloud [Parameter()] [System.Uri] $CustomCloudArmEndpoint, # URI of an HTTP proxy server [Parameter(ParameterSetName = 'Edge')] [System.Uri] $Proxy, # Azure Resource Manager endpoint URI for custom cloud [Parameter(ParameterSetName = 'Edge')] [System.Management.Automation.PSCredential] $ProxyCredential, # URI to test the proxy server [Parameter(ParameterSetName = 'Edge')] [System.Uri] $ExternalUri = 'https://management.azure.com/metadata/endpoints?api-version=2015-01-01', # List of additional URLs to test [Parameter(ParameterSetName = 'Edge')] [Parameter(ParameterSetName = 'Hub')] [System.Uri[]] $CustomUrl, # Windows Update Services server URI [Parameter(ParameterSetName = 'Edge')] [System.Uri[]] $WindowsUpdateServer = @( 'http://windowsupdate.microsoft.com' 'http://update.microsoft.com' 'https://update.microsoft.com' 'http://download.windowsupdate.com' 'http://download.microsoft.com' 'https://download.microsoft.com' 'http://go.microsoft.com' ), # External Hyper-V Switch Name on the HLH [Parameter(ParameterSetName = 'HLH')] [System.String] $VirtualSwitchName, # No Uplinks required, if the P2P interfaces do not ping to the Border this is the override to use. [Parameter(ParameterSetName = 'HLH')] [switch] $NoUplinksRequired, # Optional parameter if you only want to execute the tests for one of the networks. [Parameter(ParameterSetName = 'HLH')] [ValidateSet('ExternalNetworkOnly', 'BmcNetworkOnly')] [System.String] $NetworkToTest, # Switch to indicate HLH mode if no other HLH-specific parameters are used [Parameter(ParameterSetName = 'HLH')] [switch] $HLH, # Directory path for log and report output [Parameter(HelpMessage = 'Directory path for log and report output')] [System.String] $OutputPath = "$env:TEMP\AzsReadinessChecker", # Remove all previous progress and create a clean report [Parameter(HelpMessage = 'Remove all previous progress and create a clean report')] [switch] $CleanReport = $false ) try { #region Initialization $Global:OutputPath = $OutputPath $validationResults = @() Write-Header -Invocation $MyInvocation -Params $PSBoundParameters # Ensure we are elevated if (Test-Elevation) { Write-AzsReadinessLog -Message ("Powershell running as Administrator. Continuing.") -Type Info } else { Write-AzsReadinessLog -Message ("Running as administrator is required for this operation. `nPlease restart PowerShell as Administrator and retry.") -Type Error -toScreen Write-AzsReadinessLog -Message ("This is because it must interact with the hypervisor and network stack.") -Type Error -toScreen throw "This operation requires elevation." } $readinessReport = Get-AzsReadinessProgress -Clean:$CleanReport $readinessReport = Add-AzsReadinessCheckerJob -Report $readinessReport $returnData = @{ Success = $true Message = $null } $u = 'admin' $p = 'YourPaSsWoRd' $factoryCred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList @($u, (ConvertTo-SecureString -String $p -AsPlainText -Force)) $defaultHubTests = @( 'LinkLayer' 'PortChannel' 'BorderUplink' 'IPConfig' 'BgpPeering' 'BgpDefaultRoute' 'DnsServer' 'PathMtu' 'TimeServer' 'SyslogServer' 'AzureEndpoint' 'AdfsEndpoint' 'Graph' 'DuplicateIP' 'DnsDelegation' ) $defaultEdgeTests = @( 'LinkLayer' 'IPConfig' 'DnsServer' 'PathMtu' 'TimeServer' 'AzureEndpoint' 'WindowsUpdateServer' 'DuplicateIP' 'DnsRegistration' ) $solutionType = $PSCmdlet.ParameterSetName $webParams = @{'TimeoutSec' = 15; 'UseBasicParsing' = $true} if ($Proxy) { $webParams += @{'Proxy' = $Proxy} if ($ProxyCredential) { $webParams += @{'ProxyCredential' = $ProxyCredential} } } Write-AzsReadinessLog -Message "Starting Azure Stack $solutionType network validation" # Determine which tests to run if (-not $RunTests -and $solutionType -in 'Edge', 'Hub') { Write-AzsReadinessLog -Message "Parameter RunTests not specified. Executing default set of tests for solution '$solutionType'." if ($solutionType -eq 'Edge') { $RunTests = $defaultEdgeTests } elseif ($solutionType -eq 'Hub') { $RunTests = $defaultHubTests } } elseif ($solutionType -eq 'HLH') { $RunTests = 'HLH', 'AzureEndpoint' } foreach ($test in $RunTests) { if (($solutionType -eq 'Edge' -and $test -notin $defaultEdgeTests) -or ($solutionType -eq 'Hub' -and $test -notin $defaultHubTests)) { Write-AzsReadinessLog -Message "Test $test is not applicable to solution 'Azure Stack $solutionType'. It will be skipped." -Type 'Warning' -ToScreen $RunTests = $RunTests -ne $test } if ($test -in $SkipTests) { Write-AzsReadinessLog -Message "Will skip test '$test'" $RunTests = $RunTests -ne $test } } if ($Proxy) { $RunTests += 'Proxy' } if ($CustomUrl) { $RunTests += 'CustomUrl' } $azureEndpointTypes = @('ARM', 'Graph', 'Login', 'ManagementService') if ($solutionType -eq 'Edge') { $azureEndpointTypes += 'DnsLoadBalancer' $azureEndpointTypes += 'AseService' $azureEndpointTypes += 'AseServiceBus' $azureEndpointTypes += 'AseStorageAccount' } Write-AzsReadinessLog -Message "Preparing to run tests: $($RunTests -join ', ')" #endregion #region ParameterValidation if ($RunTests -contains 'BorderUplink') { $layer3 = $true Write-AzsReadinessLog -Message "Validating prerequisites" if ((Get-WindowsFeature | Where-Object {$_.Name -in 'Hyper-V', 'Hyper-V-PowerShell' -and $_.Installed}).Count -eq 2) { Write-AzsReadinessLog -Message 'Hyper-V features are installed' } else { Write-AzsReadinessLog -Message 'Required Hyper-V features are not installed' -Type 'Error' -ToScreen $paramValidation = $false } if (-not (Test-Path -Path $VirtualRouterImagePath -PathType Leaf)) { Write-AzsReadinessLog -Message 'Path to TOR image not found. Provide the correct value to the -VirtualRouterImagePath parameter.' -Type 'Error' -ToScreen $paramValidation = $false } } if ($RunTests -contains 'BorderUplink' -or $RunTests -contains 'HLH') { if (-not (Get-Module -ListAvailable -Name 'Posh-SSH')) { Write-AzsReadinessLog -Message 'Posh-SSH PowerShell module is not installed' -Type 'Error' -ToScreen $paramValidation = $false } } Write-AzsReadinessLog -Message "Validating input parameters" -ToScreen $paramValidation = $true if ($DeploymentDataPath) { Write-AzsReadinessLog -Message "Validating parameter 'DeploymentDataPath' value '$DeploymentDataPath'" if (Test-Path -Path $DeploymentDataPath -PathType Leaf) { Write-AzsReadinessLog -Message "Reading deployment configuration file '$DeploymentDataPath'" try { $deploymentData = Get-Content -Path $DeploymentDataPath -Raw | ConvertFrom-Json $iron = $deploymentData.ScaleUnits.DeploymentData.IsFeArchitecture $labScenario = $DeploymentData.ConfigData.InputData.IsLabScenario $cloudID = Get-CloudID -DeploymentData $deploymentData $dvmIP = $DeploymentData.ScaleUnits.DeploymentData.DeploymentVM.IPAddress $externalDnsZone = "$($deploymentData.ScaleUnits.DeploymentData.RegionName).$($deploymentData.ScaleUnits.DeploymentData.ExternalDomainFQDN)" $DnsServer = $deploymentData.ScaleUnits.DeploymentData.DnsForwarder $TimeServer = $deploymentData.ScaleUnits.DeploymentData.TimeServer $syslogServer = $deploymentData.ConfigData.InputData.Cloud.SyslogServerIPv4Address $switchData = $deploymentData.ConfigData.EnvironmentData.Switch $ipConfigData = $deploymentData.ConfigData.IPConfigData $p2pNetworks = $ipConfigData | Where-Object {$_.Name -like "P2P_Rack00/B?_To_Rack??/*"} $loopbackNetworks = $ipConfigData | Where-Object {$_.Name -like "Loopback_Rack??/Tor?"} $switchMgmtNetworks = $ipConfigData | Where-Object {$_.Name -like "Rack??-SwitchMgmt"} $bmcNetwork = $ipConfigData | Where-Object {$_.Name -eq "Rack01-BmcMgmt"} $externalNetwork = $ipConfigData | Where-Object {$_.Name -eq "$cloudID-External-VIPS"} $externalTestIP = Get-IPAddressOffset -SubnetAddress $externalNetwork.IPv4NetworkAddress -Offset 15 $routing = $deploymentData.ConfigData.InputData.BorderConnectivity $torSwitchBgpPeerIP = $deploymentData.ScaleUnits.DeploymentData.TORSwitchBGPPeerIP $torSwitchBgpAsn = $deploymentData.ScaleUnits.DeploymentData.TORSwitchBGPASN $muxBgpAsn = $deploymentData.ScaleUnits.DeploymentData.SoftwareBGPASN $skipDupTest = ($p2pNetworks.Assignments | Where-Object {$_.Name -like 'Rack??/*'}).IPv4Address $skipDupTest += ($loopbackNetworks.Assignments).IPv4Address $skipDupTest += ($switchMgmtNetworks.Assignments).IPv4Address $skipDupTest += ($bmcNetwork.Assignments | Where-Object {$_.Name -like '*DVM*' -or $_.Name -eq 'Gateway'}).IPv4Address $skipDupTest += $externalNetwork.IPv4Gateway $skipDupTest += $externalTestIP $addressRanges = @() $routing = $null $edgeFirewall = $true foreach ($supernet in ($deploymentData.ConfigData.InputData.Supernets | Where-Object {$_.Name -notlike "*-Private"}).IPv4) { $addressRanges += "$($supernet.Network)-$($supernet.Broadcast)" } if ($deploymentData.ScaleUnits.DeploymentData.UseAdfs -eq 'Selected') { Write-AzsReadinessLog -Message 'Azure Endpoint test will be skipped because this environment uses ADFS identity provider' $adfsEndpoint = $deploymentData.ScaleUnits.DeploymentData.ADFSMetadataUri $adfsForestFqdn = $deploymentData.ScaleUnits.DeploymentData.ADFSForestFQDN $RunTests = $RunTests -ne 'AzureEndpoint' } else { Write-AzsReadinessLog -Message 'ADFS test will be skipped because this environment uses Azure AD identity provider' $AzureEnvironment = $deploymentData.ScaleUnits.DeploymentData.InfraAzureEnvironment $RunTests = $RunTests -ne 'AdfsEndpoint' $RunTests = $RunTests -ne 'Graph' } if ($layer3) { if ($routing -ne 'BGP Routing') { Write-AzsReadinessLog -Message 'BGP tests will be skipped because this environment uses static routing' $RunTests = $RunTests -ne 'BgpPeering' $RunTests = $RunTests -ne 'BgpDefaultRoute' } if ($switchData.Type -notcontains 'Edge') { Write-AzsReadinessLog -Message 'Port Channel test will be skipped because this environment does not use Edge firewalls' $edgeFirewall = $false $RunTest = $RunTest -ne 'PortChannel' } } } catch { Write-AzsReadinessLog -Message "Unable to read JSON from '$DeploymentDataPath'" -Type 'Error' -ToScreen $paramValidation = $false } } else { Write-AzsReadinessLog -Message "Deployment Data file not found: $DeploymentDataPath" -Type 'Error' -ToScreen $paramValidation = $false } } # Some Azure Stack Hub tests only apply in layer-3 scenarios if ($solutionType -eq 'Hub') { if ($layer3) { $RunTests = $RunTests -ne 'IPConfig' } else { $RunTests = $RunTests -ne 'BgpPeering' $RunTests = $RunTests -ne 'BgpDefaultRoute' } } if ($RunTests -contains 'DnsServer') { if ($DnsServer) { Write-AzsReadinessLog -Message "Validating parameter 'DnsServer' value '$DnsServer'" foreach ($serverAddress in $DnsServer) { Write-AzsReadinessLog -Message "Parsing DNS server value '$serverAddress'" $parseAddress = $null if ([System.Net.IPAddress]::TryParse($serverAddress, [ref]$parseAddress)) { Write-AzsReadinessLog -Message "'$serverAddress' is a valid IP address" } else { Write-AzsReadinessLog -Message "'$serverAddress' is not a valid IP address" -Type 'Error' -ToScreen $paramValidation = $false } } } else { Write-AzsReadinessLog -Message "Parameter missing: DnsServer" -Type 'Error' -ToScreen $paramValidation = $false } Write-AzsReadinessLog -Message "Validating parameter 'DnsName' value '$DnsName'" if ([System.Uri]::CheckHostName($DnsName) -eq [System.UriHostNameType]::Dns) { Write-AzsReadinessLog -Message "'$DnsName' is a valid DNS name" } else { Write-AzsReadinessLog -Message "'$DnsName' is not a valid DNS name" -Type 'Error' -ToScreen $paramValidation = $false } } if ($RunTest -contains 'PathMtu') { Write-AzsReadinessLog -Message "Validating parameter 'MtuTestDestination' value '$MtuTestDestination'" if ([System.Uri]::CheckHostName($MtuTestDestination) -in @([System.UriHostNameType]::Dns, [System.UriHostNameType]::IPv4)) { Write-AzsReadinessLog -Message "'$MtuTestDestination' is a valid address of type '$([System.Uri]::CheckHostName($MtuTestDestination))'" } else { Write-AzsReadinessLog -Message "'$MtuTestDestination' is not a valid IP address or DNS name" -Type 'Error' -ToScreen $paramValidation = $false } } if ($RunTests -contains 'TimeServer') { if ($TimeServer) { Write-AzsReadinessLog -Message "Validating parameter 'TimeServer' value '$TimeServer'" foreach ($ntpAddress in $TimeServer) { Write-AzsReadinessLog -Message "Parsing time server value '$ntpAddress'" if ([System.Uri]::CheckHostName($ntpAddress) -in @([System.UriHostNameType]::Dns, [System.UriHostNameType]::IPv4)) { Write-AzsReadinessLog -Message "'$ntpAddress' is a valid address of type '$([System.Uri]::CheckHostName($ntpAddress))'" } else { Write-AzsReadinessLog -Message "'$ntpAddress' is not a valid IP address or DNS name" -Type 'Error' -ToScreen $paramValidation = $false } } } else { Write-AzsReadinessLog -Message 'Parameter missing: TimeServer' -Type 'Error' -ToScreen $paramValidation = $false } } if ($RunTests -contains 'SyslogServer') { if ($syslogServer) { Write-AzsReadinessLog -Message "Validating parameter 'SyslogServer' value '$syslogServer'" $parseAddress = $null if ([System.Net.IPAddress]::TryParse($syslogServer, [ref]$parseAddress)) { Write-AzsReadinessLog -Message "'$syslogServer' is a valid IP address" } else { Write-AzsReadinessLog -Message "'$syslogServer' is not a valid IP address" -Type 'Error' -ToScreen $paramValidation = $false } } else { Write-AzsReadinessLog -Message 'Parameter Syslog Server is not specified in the deployment configuration file. The Syslog Server test will be skipped.' $RunTests = $RunTests -ne 'SyslogServer' } } if ($RunTests -contains 'DuplicateIP') { if ($solutionType -eq 'Edge') { if ($ComputeIPs) { Write-AzsReadinessLog -Message "Validating parameter 'ComputeIPs' value '$ComputeIPs'" $addressRanges = $ComputeIPs } else { Write-AzsReadinessLog -Message "Parameter missing: ComputeIPs. Use -SkipTest DuplicateIP to skip the test" -Type 'Error' -ToScreen $paramValidation = $false } } foreach ($addressRange in $addressRanges) { $rangeSplit = $addressRange.Split('-') if ($rangeSplit.Count -ne 2) { Write-AzsReadinessLog -Message "Incorrect parameter value: '$addressRange'. Provide the value in the following format: 'X.X.X.X-Y.Y.Y.Y" -Type 'Error' -ToScreen $paramValidation = $false } $bgnIp = $rangeSplit[0].Trim() $endIp = $rangeSplit[1].Trim() $parseAddress = $null foreach ($ipAddress in @($bgnIp, $endIp)) { if (-not [System.Net.IPAddress]::TryParse($ipAddress, [ref]$parseAddress)) { Write-AzsReadinessLog -Message "'$ipAddress' is not a valid IP address" -Type 'Error' -ToScreen $paramValidation = $false } } if ($paramValidation) { # This double conversion is done to reverse the byte order of the integer Address property $bgnIpInt = ([System.Net.IPAddress][System.String]([System.Net.IPAddress]$bgnIp).Address).Address $endIpInt = ([System.Net.IPAddress][System.String]([System.Net.IPAddress]$endIp).Address).Address if ($endIpInt -lt $bgnIpInt) { Write-AzsReadinessLog -Message "Incorrect parameter value: '$addressRange'. The beginning of range cannot be lower than the end of range." -Type 'Error' -ToScreen $paramValidation = $false } } } } if ($RunTests -contains 'Proxy') { Write-AzsReadinessLog -Message "Validating parameter 'Proxy' value '$Proxy'" $paramValidation = $paramValidation -and (Test-Uri -Uri $Proxy -SupportedProtocols 'http') $paramValidation = $paramValidation -and (Test-Uri -Uri $ExternalUri) } if ($RunTests -contains 'AzureEndpoint' -and $AzureEnvironment -in @('AzureUSSec', 'AzureUSNat', 'CustomCloud')) { Write-AzsReadinessLog -Message "Validating parameter 'CustomCloudArmEndpoint' value '$CustomCloudArmEndpoint'" if (-not $CustomCloudArmEndpoint) { Write-AzsReadinessLog -Message 'Parameter missing: CustomCloudArmEndpoint' -Type 'Error' -ToScreen $paramValidation = $false } $paramValidation = $paramValidation -and (Test-Uri -Uri $CustomCloudArmEndpoint) } if ($RunTests -contains 'AdfsEndpoint') { Write-AzsReadinessLog -Message "Validating parameter 'ADFSEndpoint' value '$adfsEndpoint'" if (-not $adfsEndpoint.EndsWith('.xml')) { Write-AzsReadinessLog -Message "ADFS metadata URI '$adfsEndpoint' does not include the path to metadata.xml. Appending the default path." $adfsEndpoint = $adfsEndpoint.TrimEnd('/') + '/FederationMetadata/2007-06/FederationMetadata.xml' Write-AzsReadinessLog -Message "New ADFS metadata URI is '$adfsEndpoint'." } $paramValidation = $paramValidation -and (Test-Uri -Uri $adfsEndpoint -SupportedProtocols 'https') } if ($RunTests -contains 'Graph') { try { if (-not (Get-WindowsFeature | Where-Object {$_.Name -eq 'RSAT-AD-PowerShell' -and $_.Installed})) { Write-AzsReadinessLog -Message 'Installing Active Directory PowerShell Tools' $null = Install-WindowsFeature -Name 'RSAT-AD-PowerShell' } Write-AzsReadinessLog -Message 'Importing Active Directory module' Import-Module -Name ActiveDirectory -WarningAction SilentlyContinue } catch { Write-AzsReadinessLog -Message 'Failed to initialize Active Directory PowerShell Tools' $paramValidation = $false } Write-AzsReadinessLog -Message "Validating parameter 'ADFSForestFQDN' value '$adfsForestFqdn'" if ([System.Uri]::CheckHostName($adfsForestFqdn) -eq [System.UriHostNameType]::Dns) { Write-AzsReadinessLog -Message "'$adfsForestFqdn' is a valid DNS name" } else { Write-AzsReadinessLog -Message "'$adfsForestFqdn' is not a valid DNS name" -Type 'Error' -ToScreen $paramValidation = $false } } if ($RunTests -contains 'CustomUrl') { Write-AzsReadinessLog -Message "Validating parameter 'CustomUrl' value '$($CustomUrl -join ', ')'" foreach ($uri in $CustomUrl) { Write-AzsReadinessLog -Message "Parsing URL value '$uri'" $paramValidation = $paramValidation -and (Test-Uri -Uri $uri) } } if ($RunTests -contains 'WindowsUpdateServer') { Write-AzsReadinessLog -Message "Validating parameter 'WindowsUpdateServer' value '$($WindowsUpdateServer -join ', ')'" foreach ($uri in $WindowsUpdateServer) { Write-AzsReadinessLog -Message "Parsing Windows Update Server value '$uri'" $paramValidation = $paramValidation -and (Test-Uri -Uri $uri) } } if ($RunTests -contains 'DnsRegistration') { if ($DeviceFqdn) { Write-AzsReadinessLog -Message "Validating parameter 'DeviceFqdn' value '$DeviceFqdn'" if ([System.Uri]::CheckHostName($DeviceFqdn) -eq [System.UriHostNameType]::Dns) { Write-AzsReadinessLog -Message "'$DeviceFqdn' is a valid DNS name" } else { Write-AzsReadinessLog -Message "'$DeviceFqdn' is not a valid DNS name" -Type 'Error' -ToScreen $paramValidation = $false } } else { Write-AzsReadinessLog -Message "Parameter missing: DeviceFqdn" -Type 'Error' -ToScreen $paramValidation = $false } } if ($RunTests -contains 'HLH') { $vSwitch = Get-VMSwitch | Where-Object {$_.SwitchType -eq 'External'} if ($VirtualSwitchName) { Write-AzsReadinessLog -Message "Validating parameter 'VirtualSwitchName' value '$VirtualSwitchName'" if ($VirtualSwitchName -notin $vSwitch.Name) { Write-AzsReadinessLog -Message "Hyper-V switch '$VirtualSwitchName' was not found" -Type 'Error' -ToScreen $paramValidation = $false } } else { if ($vSwitch.Count -ne 1) { Write-AzsReadinessLog -Message "Multiple Hyper-V external switches exist: $($vSwitch.Name -join ','). Parameter VirtualSwitchName is required." -Type 'Error' -ToScreen $paramValidation = $false } else { $VirtualSwitchName = $vSwitch.Name } } } if (-not $paramValidation) { throw 'Parameter validation failed' } if ($solutionType -in 'Edge', 'Hub') { Write-AzsReadinessLog -Message "The following tests will be executed: $($RunTests -join ', ')" -ToScreen } #endregion #region Main $cmdletRestart = Invoke-CmdletRestart -Parameters $PSBoundParameters -FIPS if ($cmdletRestart.Restart) { Write-AzsReadinessLog -Message "Restarting cmdlet '$($MyInvocation.MyCommand.Name)'." & powershell.exe -NoLogo -NoProfile -EncodedCommand $cmdletRestart.EncodedCommand } else { Write-AzsReadinessLog -Message "Validating Azure Stack $solutionType Network Readiness" -ToScreen if ($RunTests -contains 'LinkLayer') { $testResult = Test-LinkLayer Write-AzsResult -In $testResult $validationResults += $testResult # If the link layer test fails, no other tests are applicable if ($testResult.Result -eq 'Fail') { throw 'No network connection' } } if ($RunTests -contains 'PortChannel') { $testResult = Test-PortChannel Write-AzsResult -In $testResult $validationResults += $testResult # If the port channel test fails, no other tests are applicable if ($testResult.Result -eq 'Fail') { throw 'Port Channel test failed. Ensure that port channels are configured on the border device' } } if ($RunTests -contains 'BorderUplink') { Write-AzsReadinessLog -Message "Starting border uplink discovery. This operation will take a few minutes." -ToScreen $testResult = Test-BorderUplink -Configuration $p2pNetworks Write-AzsResult -In $testResult $validationResults += $testResult if ($p2pNetworks.Name -like '*/Edge?') { Write-AzsReadinessLog -Message "Configuring LACP teams for network tests over port channel" $borderUplinks = @() $pcNum = 1 foreach ($p2pNet in $p2pNetworks | Where-Object {$_.Name -like '*/Edge1'}) { if ($pcMembers = ($testResult.OutputObject | Where-Object {$_.Subnet.Name -eq $p2pNet.Name}).NetAdapter.Name) { $pcName = "port-channel$pcNum" Write-AzsReadinessLog -Message "Creating LACP team '$pcName' with interfaces '$($pcMembers -join ',')'" $null = New-NetLbfoTeam -Name $pcName -TeamMembers $pcMembers -TeamingMode Lacp -Confirm:$false Start-Sleep -Seconds 5 $pcNic = Get-NetAdapter | Where-Object {$_.Name -eq $pcName} $borderUplinks += [PSCustomObject]@{NetAdapter = $pcNic; Subnet = $p2pNet} $pcNum++ } } } else { $borderUplinks = $testResult.OutputObject | Where-Object {$_.PeerConnected} $torIpAssigned = $borderUplinks | Where-Object {$_.WasDiscovered} } # If the border uplink test fails, no other tests are applicable if (($testResult.Result -eq 'OK') -or ($testResult.Result -eq 'WARNING' -and $routing -eq 'BGP Routing')) { Write-AzsReadinessLog -Message "Starting the virtual router for all subsequent tests. This operation will take a few minutes." -ToScreen if ($routing -eq 'BGP Routing') { $uplinkCount = 1 } elseif ($routing -eq 'Static Routing') { $uplinkCount = $borderUplinks.Count } $virtualRouterSsh = Start-VirtualRouter -ImagePath $VirtualRouterImagePath -Credential $factoryCred -DnsServer $DnsServer -UplinkCount $uplinkCount -DeploymentData $deploymentData if (-not $virtualRouterSsh.Connected) { throw 'Failed to start the virtual router. Check the AzsReadinessChecker log file for details.' } } elseif ($testResult.Result -eq 'WARNING' -and $routing -eq 'Static Routing') { throw 'All border uplinks must be connected when using static routing' } else { throw 'Border connectivity test failed' } } if ($RunTests -contains 'IPConfig') { $testResult = Test-IPConfig Write-AzsResult -In $testResult $validationResults += $testResult # If the ip config test fails, no other tests are applicable if ($testResult.Result -eq 'Fail') { throw 'Invalid IP configuration' } } # If more than one interface is connected, all connectivity tests are performed from each interface, with all other interfaces shut down # When using static routing, the tests are performed once with all interfaces active if ($routing -eq 'BGP Routing') { $uplinksToTest = $borderUplinks } elseif ($solutionType -eq 'HLH') { $uplinksToTest = $null } else { $uplinksToTest = @('AllAtOnce') } $ipRoutes = Get-NetRoute -AddressFamily IPv4 | Where-Object {$_.NextHop -ne '0.0.0.0'} $routedInterfaces = Get-NetIPInterface -AddressFamily IPv4 | Where-Object {$_.ConnectionState -eq 'Connected' -and $_.InterfaceIndex -in $ipRoutes.InterfaceIndex} $netAdapters = Get-NetAdapter | Where-Object {$_.InterfaceIndex -in $routedInterfaces.InterfaceIndex} | Select-Object -Unique | Sort-Object -Property 'Name' $disabledNetAdapters = @() foreach ($uplink in $uplinksToTest) { if ($layer3) { if ($edgeFirewall) { $localAsn = ($switchData | Where-Object {$_.Type -eq 'Edge'} | Select-Object -First 1).Asn } else { $localAsn = ($switchData | Where-Object {$_.Type -eq 'TOR1'}).Asn } $remoteAsn = ($switchData | Where-Object {$_.Type -eq 'Border'} | Select-Object -First 1).Asn $interfaceConfiguration = @() $bgpNeighborConfiguration = @() $ethNumber = 8 # Connect one uplink to the virtual router in BGP scenario # Connect all uplinks in the static routing scenario if ($routing -eq 'BGP Routing') { $uplinkConfiguration = $uplink } else { $uplinkConfiguration = $borderUplinks } foreach ($connectUplink in $uplinkConfiguration) { $localAssignment = $connectUplink.Subnet.Assignments | Where-Object {$_.Offset -eq 2} $remoteAssignment = $connectUplink.Subnet.Assignments | Where-Object {$_.Offset -eq 1} Write-AzsReadinessLog -Message "UPLINK: '$($connectUplink.Subnet.Name)', Local IP: '$($localAssignment.IPv4Address)', Remote IP: '$($remoteAssignment.IPv4Address)', Interface: '$($connectUplink.Netadapter.Name)'" -ToScreen $vnicName = "Ethernet$ethNumber" Write-AzsReadinessLog -Message "Connecting network adapter '$($connectUplink.Netadapter.Name)' to the virtual router port $vnicName" Connect-VirtualRouterUplink -NetAdapterInterfaceDescription $connectUplink.NetAdapter.InterfaceDescription -VNicName $vnicName $interfaceConfiguration += [PSCustomObject]@{ Name = $vnicName IPAddress = $localAssignment.IPv4Address PrefixLength = $connectUplink.Subnet.IPv4MaskBits } $bgpNeighborConfiguration += [PSCustomObject]@{ Name = $remoteAssignment.Name LocalAddress = $localAssignment.IPv4Address RemoteAddress = $remoteAssignment.IPv4Address RemoteAsn = $remoteAsn } $ethNumber = $ethNumber + 4 } $loopbackAddress = ($loopbackNetworks.Assignments | Where-Object {$_.Name -eq $localAssignment.Name}).IPv4Address Write-AzsReadinessLog -Message "Applying configuration to the virtual router" Set-VirtualRouterConfig ` -SSHSession $virtualRouterSsh ` -Credential $factoryCred ` -Clear ` -LocalAsn $localAsn ` -LoopbackAddress $loopbackAddress ` -InterfaceConfiguration $interfaceConfiguration ` -BgpNeighborConfiguration $bgpNeighborConfiguration ` -BmcNetworkAddress $bmcNetwork.IPv4Gateway ` -BmcNetworkPrefixLength $bmcNetwork.IPv4MaskBits ` -ExternalNetworkAddress $externalNetwork.IPv4Gateway ` -ExternalNetworkPrefixLength $externalNetwork.IPv4MaskBits if ($RunTests -contains 'BgpPeering') { $testResult = Test-BgpPeering -SSHSession $virtualRouterSsh -Credential $factoryCred Write-AzsResult -In $testResult $validationResults += $testResult if ($testResult.Result -ne 'OK') { Write-AzsReadinessLog -Message "Skipping other tests for this uplink as not applicable" continue } } if ($RunTests -contains 'BgpDefaultRoute') { $testResult = Test-BgpDefaultRoute -SSHSession $virtualRouterSsh -Credential $factoryCred Write-AzsResult -In $testResult $validationResults += $testResult if ($testResult.Result -ne 'OK') { Write-AzsReadinessLog -Message "Skipping other tests for this uplink as not applicable" continue } } } foreach ($netAdapter in $netAdapters) { if ($netAdapters.Count -gt 1) { Write-AzsReadinessLog -Message "Using network adapter '$($netAdapter.Name)'" -ToScreen } foreach ($otherNetAdapter in $netAdapters | Where-Object {$_.InterfaceIndex -ne $netAdapter.InterfaceIndex}) { Write-AzsReadinessLog -Message "Disabling network adapter name '$($otherNetAdapter.Name)', description '$($otherNetAdapter.InterfaceDescription)'" Disable-NetAdapter -Name $otherNetAdapter.Name -Confirm:$false $disabledNetAdapters += $otherNetAdapter.Name } if ($RunTests -contains 'DnsServer') { foreach ($serverAddress in $DnsServer) { $testResult = Test-DnsServer -DnsServer $serverAddress -ResolveName $DnsName Write-AzsResult -In $testResult $validationResults += $testResult } } if ($RunTests -contains 'PathMtu') { $testResult = Test-PathMtu -Destination $MtuTestDestination Write-AzsResult -In $testResult $validationResults += $testResult } if ($RunTests -contains 'TimeServer') { foreach ($serverAddress in $TimeServer) { $testResult = Test-TimeServer -TimeServer $serverAddress Write-AzsResult -In $testResult $validationResults += $testResult } } if ($RunTests -contains 'SyslogServer') { $testResult = Test-SyslogServer -SyslogServer $syslogServer Write-AzsResult -In $testResult $validationResults += $testResult } if ($RunTests -contains 'Proxy') { $testResult = Test-ProxyServer -Proxy $Proxy -ProxyCredential $ProxyCredential -TestUri $ExternalUri Write-AzsResult -In $testResult $validationResults += $testResult } if ($RunTests -contains 'AzureEndpoint') { foreach ($endpointType in $azureEndpointTypes) { $testResult = Test-AzureEndpoint -AzureEnvironment $AzureEnvironment -CustomCloudArmEndpoint $CustomCloudArmEndpoint -EndpointType $endpointType Write-AzsResult -In $testResult $validationResults += $testResult } } if ($RunTests -contains 'AdfsEndpoint') { $testResult = Test-AdfsEndpoint -MetadataUri $adfsEndpoint Write-AzsResult -In $testResult $validationResults += $testResult } if ($RunTests -contains 'Graph') { $testResult = Test-Graph -ForestFqnd $adfsForestFqdn Write-AzsResult -In $testResult $validationResults += $testResult } if ($RunTests -contains 'CustomUrl') { foreach ($serverAddress in $CustomUrl) { $testResult = Test-CustomUrl -ServerUri $serverAddress Write-AzsResult -In $testResult $validationResults += $testResult } } if ($RunTests -contains 'WindowsUpdateServer') { foreach ($serverAddress in $WindowsUpdateServer) { $testResult = Test-WindowsUpdateServer -ServerUri $serverAddress Write-AzsResult -In $testResult $validationResults += $testResult } } if ($disabledNetAdapters) { Write-AzsReadinessLog -Message "Re-enabling previously disabled network adapters" Enable-NetAdapter -Name $disabledNetAdapters Start-Sleep -Seconds 5 $disabledNetAdapters = @() } } } if ($netAdapters.Count -gt 1) { Write-AzsReadinessLog -Message "Testing network services configuration" -ToScreen } if ($RunTests -contains 'DuplicateIP') { $testResult = Test-DuplicateIP -AddressRanges $addressRanges -SkipAddresses $skipDupTest Write-AzsResult -In $testResult $validationResults += $testResult } if ($RunTests -contains 'DnsRegistration') { foreach ($dnsRecord in @('', 'login.', 'management.', '*.blob.', 'compute.')) { $testResult = Test-DnsRegistration -DnsServer $DnsServer -DnsRecord ($dnsRecord + $DeviceFqdn) Write-AzsResult -In $testResult $validationResults += $testResult } } if ($RunTests -contains 'DnsDelegation') { $testResult = Test-DnsDelegation -DnsDomain $externalDnsZone -IPv4Address $externalTestIP Write-AzsResult -In $testResult $validationResults += $testResult } if ($solutionType -eq 'HLH') { $sonicVMname = "$cloudID-SonicMux" # Execute the BMC network tests if ($NetworkToTest -ne "ExternalNetworkOnly") { if (Get-VM | Where-Object {$_.Name -eq $sonicVMname}) { Write-AzsReadinessLog -Message "Removing the existing virtual machine $sonicVMname" Stop-VM -Name $sonicVMname -TurnOff -Force Remove-VM -Name $sonicVMname -Force } $testResult = Test-BmcNetworkOnHlh -DeploymentDataPath $DeploymentDataPath -VirtualSwitchName $VirtualSwitchName -NoUplinksRequired $NoUplinksRequired -CustomCloudArmEndpoint $CustomCloudArmEndpoint Write-AzsResult -In $testResult $validationResults += $testResult $returnData.BmcNetwork = @{ Success = $testResult.OutputObject.Success Message = $testResult.OutputObject.Message } foreach ($subtestResult in $testResult.OutputObject.ValidationResults) { $validationResults += $subtestResult } # If the BMC network test fails, do not continue if (-not $testResult.OutputObject.Success) { throw $testResult.OutputObject.Message } Write-AzsReadinessLog -Message $testResult.OutputObject.Message } if ($NetworkToTest -eq "BmcNetworkOnly") { Write-AzsReadinessLog -Message "Testing is complete for $NetworkToTest. Full network validation is required prior to deployment." -ToScreen -Type 'Warning' Write-AzsReadinessLog -Message "Execute the test again without using the -NetworkToTest paramter. See log for further details." -ToScreen -Type 'Warning' } else { # Prepare environment for External network tests $vmStatus = Get-VM | Where-Object {$_.Name -eq $sonicVMname} $checkForVnic = Get-VMNetworkAdapter -ManagementOS | Where-Object {$_.Name -eq "$cloudID-AzSPreValNic-DVM"} if ($checkForVnic) { $checkForVnic | Remove-VMNetworkAdapter } # Prepare the Sonic VM if ($vmStatus.State -eq "Running") { Write-AzsReadinessLog -Message "$sonicVMname is already up and running." } else { if ($vmStatus -and $vmStatus.State -ne "Running") { Write-AzsReadinessLog -Message "Existing virtual machine $sonicVMname needs to be recreated and will be deleted." Stop-VM -Name $sonicVMname -TurnOff -Force Remove-VM -Name $sonicVMname -Force } $null = Get-SSHTrustedHost -WarningAction SilentlyContinue | Remove-SSHTrustedHost Write-AzsReadinessLog -Message "Preparing to start the Sonic VM to test the External Network." -ToScreen $sonicSession = Start-VirtualRouter -VMName $sonicVMname -DeploymentData $deploymentData -ImagePath $VirtualRouterImagePath -Credential $factoryCred -HlhVirtualSwitchName $VirtualSwitchName if (-not $sonicSession.Connected) { throw 'Failed to start the Sonic VM. Check the AzsReadinessChecker log file for details.' } # Create the <CloudID>-TestData.json file and configure SONiC VM $testData = Set-TestDataForSonicVM -DeploymentData $deploymentData -CustomCloudArmEndpoint $CustomCloudArmEndpoint -OutputPath $OutputPath $interfaceConfiguration = [PSCustomObject]@{ Name = "Ethernet0" IPAddress = $dvmIP PrefixLength = $bmcNetwork.IPv4MaskBits } $bgpNeighborConfiguration = @( [PSCustomObject]@{ Name = "Tor1" LocalAddress = $dvmIP RemoteAddress = $torSwitchBGPPeerIP[0] RemoteAsn = $torSwitchBGPASN }, [PSCustomObject]@{ Name = "Tor2" LocalAddress = $dvmIP RemoteAddress = $torSwitchBGPPeerIP[1] RemoteAsn = $torSwitchBGPASN } ) Set-VirtualRouterConfig ` -SSHSession $sonicSession ` -Credential $factoryCred ` -Clear ` -LocalAsn $muxBgpAsn ` -LoopbackAddress $externalTestIP ` -InterfaceConfiguration $interfaceConfiguration ` -BgpNeighborConfiguration $bgpNeighborConfiguration ` -TestDataPath $OutputPath } # Create SSH session via the data plane to ensure the Sonic VM is ready. $ssh = Assert-VirtualRouterSession -ComputerName $dvmIP -Credential $factoryCred if (-not $ssh) { $message = "Unable to establish SSH connection with the Sonic VM ($sonicVMname) after $retry attempts." Write-AzsReadinessLog -Message $message -Type 'Error' throw $message } # Execute the External Network Tests Write-AzsReadinessLog -Message "Executing External Network Tests using IP $externalTestIP." -ToScreen $timeStamp = Get-Date -f yyyy-MM-dd-HHmmss $sonicFilesPath = Join-Path -Path $OutputPath -ChildPath "$sonicVMname.$timeStamp" $testResult = Test-ExternalNetworkOnHlh -VirtualRouterSession $ssh -CloudID $cloudID -Credential $factoryCred -LogDestinationPath $sonicFilesPath # Interpret and display the results and write detailed messages in the log foreach ($tor in $testResult.OutputObject.PSObject.Properties | Where-Object {$_.Name -match 'tor'}) { if ($failureDetail = $tor.Value.Message) { $torTestResult = [PSCustomObject]@{ Test = $tor.Name Result = if ($tor.Value.Success) {"OK"} else {"Fail"} FailureDetail = $failureDetail OutputObject = $null } Write-AzsResult -in $torTestResult $validationResults += $torTestResult } foreach ($subtest in $tor.Value.PSObject.Properties) { $torSubtestResult = [PSCustomObject]@{ Test = "$($tor.Name)_$($subtest.Name)" Result = if ($subtest.Value.Success) {"OK"} else {"Fail"} FailureDetail = if ($subtest.Value.Success) {$null} else {$subtest.Value.Message -match '^FAIL:'} OutputObject = $null } Write-AzsResult -in $torSubtestResult $validationResults += $torSubtestResult foreach ($message in $subtest.Value.Message) { $messageType = if ($message -match "^FAIL:") {'Error'} else {'Info'} Write-AzsReadinessLog -Message $message -Type $messageType } } } # Display the overall result for the External network Write-AzsReadinessLog -Message "External Network Test: Success=$($testResult.OutputObject.Success)" Write-AzsReadinessLog -Message "External Network Test: Message=$($testResult.OutputObject.Message)" Write-AzsResult -In $testResult $validationResults += $testResult $returnData.ExternalNetwork = @{ Success = $testResult.OutputObject.Success Message = $testResult.OutputObject.Message } if ($NetworkToTest) { Write-AzsReadinessLog -Message "Testing is complete for $NetworkToTest. Full network validation is required prior to deployment." -ToScreen -Type 'Warning' Write-AzsReadinessLog -Message "Execute the test again without using the -NetworkToTest paramter. See log for further details." -ToScreen -Type 'Warning' $returnData.Message = $testResult.OutputObject.Message } if ($returnData.ExternalNetwork.Success -and $returnData.BmcNetwork.Success) { $returnData.Success = $true $returnData.Message = "The network is ready for Azure Stack Hub Deployment." } } } } #endregion } catch { $exception = $_ $file = $exception.InvocationInfo.ScriptName $line = $exception.InvocationInfo.ScriptLineNumber $exceptionMessage = $exception.Exception.Message $message = "$file : $line >> $exceptionMessage" Write-AzsReadinessLog -Message "Network validation failed with error: $message" -Type 'Error' -ToScreen $returnData.Success = $false $returnData.Message = $exceptionMessage } finally { if ($disabledNetAdapters) { Write-AzsReadinessLog -Message "Re-enabling previously disabled network adapters" -ToScreen Enable-NetAdapter -Name $disabledNetAdapters } if ($layer3) { $null = Stop-VirtualRouter Start-Sleep -Seconds 5 } elseif ($sonicVMname -and $returnData.Success) { $null = Stop-VirtualRouter -VMName $sonicVMname -CloudID $cloudID -IsLabScenario $labScenario Start-Sleep -Seconds 5 } foreach ($ipAssigned in $torIpAssigned) { $ip = ($ipAssigned.Subnet.Assignments | Where-Object {$_.Offset -eq 2}).IPv4Address $netAdapter = $ipAssigned.NetAdapter Write-AzsReadinessLog -Message "Removing previously assigned IP address $ip from the interface $($netAdapter.Name)" Remove-NetIPAddress -InterfaceIndex $netAdapter.InterfaceIndex -IPAddress $ip -Confirm:$false } foreach ($lacp in (Get-NetLbfoTeam | Where-Object {$_.Name -like "port-channel*"})) { Write-AzsReadinessLog -Message "Removing LACP team $($lacp.Name)" Get-NetLbfoTeam | Where-Object {$_.Name -eq $lacp.Name} | Remove-NetLbfoTeam -Confirm:$false } try { Get-SSHSession | Remove-SSHSession Get-SSHTrustedHost -WarningAction SilentlyContinue | Remove-SSHTrustedHost } catch {} if (Get-Module -ListAvailable | Where-Object {$_.Name -eq 'dhcpserver'}) { Write-AzsReadinessLog -Message "Cleaning up IP's that do not ping in the DHCP scope to avoid exhaustion." Invoke-ScopeCleanup -DeploymentData $deploymentData } else { Write-AzsReadinessLog -Message 'DHCP server module is not installed, no DHCP scope cleanup required.' } # Write results to readiness report $helpMessage = $validationResults | Where-Object {$_.Result -ne 'OK'} | Foreach-Object {"$($_.Test): $($_.FailureDetail)"} if ($helpMessage) { Write-AzsResult -In $helpMessage -AdditionalHelpUrl 'http://aka.ms/azsnrc' } $readinessReport.NetworkValidation = $validationResults $readinessReport = Close-AzsReadinessCheckerJob -Report $readinessReport Write-AzsReadinessProgress -Report $readinessReport Write-AzsReadinessReport -Report $readinessReport Write-Footer -invocation $MyInvocation # Return aggregate result if ($exception -or $validationResults.Result -contains 'Fail') { $returnData.Success = $false $messageType = 'Error' } elseif ($validationResults.Result -contains 'Warninig') { $messageType = 'Warning' } else { $messageType = 'Success' } if ($cmdletRestart.Restart -and $cmdletRestart.FIPSMode) { Write-AzsReadinessLog -Message 'Reset FIPS mode after cmdlet restart completion.' $null = Reset-FIPSMode -Enabled $true } Write-AzsReadinessLog -Message "Network validation completed with the summary status of $messageType" -Type $messageType -ToScreen } return $returnData } <# .SYNOPSIS Verifies link layer connectivity. .DESCRIPTION Lists all network adapters and their link state and speed. Checks that at least one Ethernet network adapter is connected to a network. #> function Test-LinkLayer { [CmdletBinding()] param () $test = 'Link Layer' Write-AzsReadinessLog -Message "Starting test '$test'" $netAdapterProperties = @( 'Name' 'InterfaceDescription' 'InterfaceIndex' 'MacAddress' 'MediaType' 'PhysicalMediaType' 'Status' 'AdminStatus' 'LinkSpeed' 'MediaConnectionState' 'ConnectorPresent' 'DriverInformation' 'DriverFileName' 'NetworkAddresses' ) try { $netAdapters = Get-NetAdapter | Select-Object -Property $netAdapterProperties $outputObject = @{'NetAdapters' = $netAdapters} foreach ($netAdapter in $netAdapters) { Write-AzsReadinessLog -Message "Found network interface: Name='$($netAdapter.Name)', Description='$($netAdapter.InterfaceDescription)', Status='$($netAdapter.Status)', LinkSpeed=$($netAdapter.LinkSpeed)'" } try { $vNics = Get-VMNetworkAdapter -ManagementOS } catch {} if ($connectedEthernet = @($netAdapters | Where-Object {$_.MediaType -eq '802.3' -and $_.Status -eq 'Up' -and $_.NetworkAddresses -notin $vNics.MacAddress})) { $result = 'OK' Write-AzsReadinessLog -Message "Found $($connectedEthernet.Count) connected physical network interfaces" } else { $result = 'Fail' $failureDetail = 'No connected ethernet adapters found' Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies LACP connectivity .DESCRIPTION Enables LACP teaming on all connected physical network adapters one at a time and returns LACP negotiation status. #> function Test-PortChannel { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.Object[]] $Configuration ) $test = 'Port Channel' Write-AzsReadinessLog -Message "Starting test '$test'" try { $outputObject = @() $result = 'OK' $ipInterfaces = Get-NetIPInterface $pNics = Get-NetAdapter | Where-Object {-not $_.Virtual -and $_.MediaType -eq '802.3' -and $_.Status -eq 'Up' -and $_.InterfaceIndex -in $ipInterfaces.InterfaceIndex} foreach ($nic in $pNics) { Write-AzsReadinessLog -Message "Testing network adapter '$($nic.Name)'" $null = New-NetLbfoTeam -Name "LACP Test $($nic.Name)" -TeamMembers $nic.Name -TeamingMode Lacp -Confirm:$false $retry = 0 do { Start-Sleep -Seconds 1 $lacpStatus = Get-NetLbfoTeam -Name "LACP Test $($nic.Name)" $retry++ } until ($lacpStatus.Status -eq 'Up' -or $retry -eq 10) if ($lacpStatus.Status -ne 'Up') { $failureDetail = 'Unable to establish LACP connection on one or more physical interfaces' $result = 'Fail' } Write-AzsReadinessLog -Message "LACP status for network adapter '$($nic.Name)' is '$($lacpStatus.Status)'" $outputObject += [PSCustomObject]@{NetAdapter = $nic; LACP = $lacpStatus} } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' } finally { Write-AzsReadinessLog -Message "Removing LACP teams" Get-NetLbfoTeam | Where-Object {$_.Name -like "LACP Test *"} | Remove-NetLbfoTeam -Confirm:$false } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies border L3 connectivity. .DESCRIPTION Lists all connected physical network adapters with default IP configuration. For each NIC, attempt to assign a P2P IP address and ping the peer address. #> function Test-BorderUplink { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.Object[]] $Configuration ) $test = 'Border Uplink' Write-AzsReadinessLog -Message "Starting test '$test'" $netAdapterProperties = @( 'Name' 'InterfaceDescription' 'InterfaceIndex' 'MacAddress' 'MediaType' 'PhysicalMediaType' 'Status' 'AdminStatus' 'LinkSpeed' 'MediaConnectionState' 'ConnectorPresent' 'DriverInformation' 'DriverFileName' 'NetworkAddresses' ) try { $outputObject = @() $matchedSubnets = @() $assignedIP = @{} $torIPs = ($Configuration.Assignments | Where-Object {$_.Offset -eq 2}).IPv4Address $netAdapters = Get-NetAdapter | Select-Object -Property $netAdapterProperties $vNics = Get-VMNetworkAdapter -ManagementOS $pNics = $netAdapters | Where-Object {$_.MediaType -eq '802.3' -and $_.Status -eq 'Up' -and $_.NetworkAddresses -notin $vNics.MacAddress} if ($Configuration.Name -like '*/Edge?') { $portChannel = $true } else { $portChannel = $false } foreach ($nic in $pNics) { Write-AzsReadinessLog -Message "Testing network adapter '$($nic.Name)'" $ping = $false $ipNic = $nic if ($portChannel) { Write-AzsReadinessLog -Message "Enabling LACP on the interface" $null = New-NetLbfoTeam -Name "LACP Test $($nic.Name)" -TeamMembers $nic.Name -TeamingMode Lacp -Confirm:$false Start-Sleep -Seconds 5 $ipNic = Get-NetAdapter -Name "LACP Test $($nic.Name)" } # Determine whether to use the currently assigned P2P IP, discover a P2P IP, or skip the interface if ($ipAddress = Get-NetIPAddress -AddressFamily IPv4 | Where-Object {$_.InterfaceIndex -eq $ipNic.InterfaceIndex} | Select-Object -First 1) { $ip = $ipAddress.IPAddress Write-AzsReadinessLog -Message "Interface IP address: $ip" if ($ipAddress.PrefixOrigin -in 'Manual', 'Dhcp') { Write-AzsReadinessLog -Message "Interface has a $($ipAddress.PrefixOrigin)-assigned IP address" if ($ip -in $torIPs) { Write-AzsReadinessLog -Message "Interface has a TOR IP address $ip assigned" $discover = $false } else { Write-AzsReadinessLog -Message "The interface's IP address is not associated with TOR configuration and will be skipped" continue } } else { Write-AzsReadinessLog -Message "The interface doesn't have a valid IP address assigned. Will use this interface for uplink discovery" $discover = $true } } else { Write-AzsReadinessLog -Message "Interface does not have IP protocol enabled. It could be attached to a Hyper-V virtual switch" continue } # If a P2P IP was previously assigned, ping the peer, otherwise iteratively assign IP addresses and try to ping the respective peer until found if ($discover) { foreach ($p2pSubnet in $Configuration | Where-Object {$_.Name -notin $matchedSubnets.Name -and $_.Name -notlike '*/Edge2'}) { $ip = ($p2pSubnet.Assignments | Where-Object {$_.Offset -eq 2}).IPv4Address $peerIp = ($p2pSubnet.Assignments | Where-Object {$_.Offset -eq 1}).IPv4Address Write-AzsReadinessLog -Message "Assigning IP address $ip to the interface $($ipNic.Name) and attempting discovery" $null = New-NetIPAddress -InterfaceIndex $ipNic.InterfaceIndex -IPAddress $ip -PrefixLength $p2pSubnet.IPv4MaskBits $assignedIP.Add($ipNic.InterfaceIndex, $ip) Start-Sleep -Seconds 5 Write-AzsReadinessLog -Message "Attempting to ping remote address $peerIp" if (Test-Connection -ComputerName $peerIp -Count 5 -Quiet) { Write-AzsReadinessLog -Message "Ping successful" $ping = $true if (-not $portChannel) { $matchedSubnets += $p2pSubnet } break } else { Write-AzsReadinessLog -Message "Peer $peerIp did not respond" Write-AzsReadinessLog -Message "Removing IP address $ip from the interface $($ipNic.Name)" Remove-NetIPAddress -InterfaceIndex $ipNic.InterfaceIndex -IPAddress $ip -Confirm:$false $assignedIP.Remove($ipNic.InterfaceIndex) Start-Sleep -Seconds 5 } } } else { $p2pSubnet = $Configuration | Where-Object {$_.Assignments.IPv4Address -contains $ip} $matchedSubnets += $p2pSubnet $peerIp = ($p2pSubnet.Assignments | Where-Object {$_.Offset -eq 1}).IPv4Address Write-AzsReadinessLog -Message "Attemping to ping remote address $peerIp" if (Test-Connection -ComputerName $peerIp -Count 5 -Quiet) { Write-AzsReadinessLog -Message "Ping successful" $ping = $true } else { Write-AzsReadinessLog -Message "Peer $peerIp did not respond" } } if ($portChannel) { Write-AzsReadinessLog -Message "Removing IP address $ip from the interface $($ipNic.Name)" Remove-NetIPAddress -InterfaceIndex $ipNic.InterfaceIndex -IPAddress $ip -Confirm:$false $assignedIP.Remove($ipNic.InterfaceIndex) Start-Sleep -Seconds 5 Write-AzsReadinessLog -Message "Disabling LACP on the interface" Get-NetLbfoTeam | Where-Object {$_.Name -eq "LACP Test $($nic.Name)"} | Remove-NetLbfoTeam -Confirm:$false } $outputObject += [PSCustomObject]@{NetAdapter = $nic; Subnet = $p2pSubnet; PeerConnected = $ping; WasDiscovered = $discover} } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' foreach ($interfaceIndex in $assignedIP.Keys) { $ip = $assignedIP.$interfaceIndex Write-AzsReadinessLog -Message "Removing IP address $ip from the interface $interfaceIndex" Remove-NetIPAddress -InterfaceIndex $interfaceIndex -IPAddress $ip -Confirm:$false Start-Sleep -Seconds 5 } } $peerConnectedCount = ($outputObject | Where-Object {$_.PeerConnected}).Count if ($peerConnectedCount -eq $Configuration.Count) { Write-AzsReadinessLog -Message "Test returned $($Configuration.Count) connected uplinks" $result = 'OK' } elseif($peerConnectedCount -eq 0) { Write-AzsReadinessLog -Message "Could not contact the border on any of the physical interfaces" $result = 'Fail' } else { $failureDetail = "Discovered the border connectivity on '$peerConnectedCount' interfaces but expected '$($Configuration.Count)'." Write-AzsReadinessLog -Message $failureDetail -Type 'Warning' $result = 'WARNING' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies IP configuration. .DESCRIPTION Lists all network adapters that have an IP address assigned. Checks that at least one Ethernet network adapter is configured with a default gateway. #> function Test-IPConfig { [CmdletBinding()] param () $test = 'IP Configuration' Write-AzsReadinessLog -Message "Starting test '$test'" $ipInterfaceProperties = @( 'InterfaceIndex' 'InterfaceAlias' 'CompartmentId' 'AddressFamily' 'Forwarding' 'ClampMss' 'Advertising' 'NlMtu' 'AutomaticMetric' 'InterfaceMetric' 'NeighborDiscoverySupported' 'NeighborUnreachabilityDetection' 'BaseReachableTime' 'ReachableTime' 'RetransmitTime' 'DadTransmits' 'DadRetransmitTime' 'RouterDiscovery' 'ManagedAddressConfiguration' 'OtherStatefulConfiguration' 'WeakHostSend' 'WeakHostReceive' 'IgnoreDefaultRoutes' #'AdvertisedRouterLifetime' 'AdvertiseDefaultRoute' 'CurrentHopLimit' 'ForceArpNdWolPattern' 'DirectedMacWolPattern' 'EcnMarking' 'Dhcp' 'ConnectionState' ) $ipAddressProperties = @( 'IPAddress' 'InterfaceIndex' 'InterfaceAlias' 'AddressFamily' 'Type' 'PrefixLength' 'PrefixOrigin' 'SuffixOrigin' 'AddressState' #'ValidLifetime' #'PreferredLifetime' 'SkipAsSource' ) $netRouteProperties = @( 'DestinationPrefix' 'InterfaceIndex' 'InterfaceAlias' 'CompartmentId' 'AddressFamily' 'NextHop' 'Publish' 'RouteMetric' 'Protocol' #'ValidLifetime' #'PreferredLifetime' ) try { $ipInterfaces = Get-NetIPInterface -AddressFamily IPv4 | Where-Object {$_.InterfaceIndex -gt 1} | Select-Object -Property $ipInterfaceProperties $ipAddresses = Get-NetIPAddress -AddressFamily IPv4 | Where-Object {$_.InterfaceIndex -gt 1} | Select-Object -Property $ipAddressProperties $ipRoutes = Get-NetRoute -AddressFamily IPv4 | Where-Object {$_.NextHop -ne '0.0.0.0'} | Select-Object -Property $netRouteProperties $defaultRoutes = @($ipRoutes | Where-Object {$_.DestinationPrefix -eq '0.0.0.0/0'}) $outputObject = @{'IPInterfaces' = $ipInterfaces; 'IPAddresses' = $ipAddresses; 'IPRoutes' = $ipRoutes} foreach ($ipInterface in $ipInterfaces) { $ipAddress = $ipAddresses | Where-Object {$_.InterfaceIndex -eq $ipInterface.InterfaceIndex} $ipRoute = $ipRoutes | Where-Object {$_.InterfaceIndex -eq $ipInterface.InterfaceIndex -and $_.NextHop -ne '0.0.0.0'} | Select-Object -Property @{n = 'Route'; e = {$_.DestinationPrefix + ' > ' + $_.NextHop}} Write-AzsReadinessLog -Message "Found IP interface: Name='$($ipInterface.InterfaceAlias)', IPAddresses='$($ipAddress.IPAddress -join ', ')', DHCP='$($ipInterface.Dhcp)', ConnectionState='$($ipInterface.ConnectionState)', Routes='$($ipRoute.Route -join ', ')'" } if ($routedInterfaces = @($ipInterfaces | Where-Object {$_.ConnectionState -eq 'Connected' -and $_.InterfaceIndex -in $ipRoutes.InterfaceIndex})) { $result = 'OK' Write-AzsReadinessLog -Message "Found $($routedInterfaces.Count) connected IP-enabled interfaces with routing configuration" if ($defaultRoutes.Count -gt 1) { $result = 'WARNING' $failureDetail = "Multiple network adapters ($($defaultRoutes.InterfaceAlias -join ', ')) are configured with a default gateway. Multiple default gateways can cause connectivity problems. Refer to this article for more information: https://docs.microsoft.com/en-us/troubleshoot/windows-server/networking/connectivity-issues-multiple-default-gateways" } } else { $result = 'Fail' $failureDetail = 'No IP-enabled network interfaces with routing configuation are found' Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Checks for duplicate IP address assignments. .DESCRIPTION Pings IP addresses in the specified range to detect if they are already being used on the network. #> function Test-DuplicateIP { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String[]] $AddressRanges, [Parameter()] [System.String[]] $SkipAddresses ) $test = 'Duplicate IP' Write-AzsReadinessLog -Message "Starting test '$test'" $netNeighborProperties = @( 'InterfaceAlias' 'InterfaceIndex' 'IPAddress' 'LinkLayerAddress' 'State' ) try { $addressList = @() $pingOutput = @() $result = 'OK' foreach ($addressRange in $AddressRanges) { $rangeSplit = $addressRange.Split('-') $bgnIp = $rangeSplit[0].Trim() $endIp = $rangeSplit[1].Trim() $bgnIpInt = ([System.Net.IPAddress][System.String]([System.Net.IPAddress]$bgnIp).Address).Address $endIpInt = ([System.Net.IPAddress][System.String]([System.Net.IPAddress]$endIp).Address).Address for ($ipInt = $bgnIpInt; $ipInt -le $endIpInt; $ipInt++) { $ipAddress = ([System.Net.IPAddress][System.String]$ipInt).IPAddressToString if ($ipAddress -notin $SkipAddresses) { $addressList += $ipAddress } } } if ($addressList.Count -ge 100) { Write-AzsReadinessLog -Message "Will ping $($addressList.Count) IP addresses to detect potential conflict. The test will take a few minutes." -ToScreen } $ping = New-Object -TypeName System.Net.NetworkInformation.Ping $timeoutMs = 100 $count = 0 foreach ($ipAddress in $addressList) { $count++ $pingResult = $ping.Send($ipAddress, $timeoutMs) $pingOutput += $pingResult | Select-Object -Property @{n = 'Address'; e = {$ipAddress}}, @{n = 'Status'; e = {$_.Status.ToString()}}, RoundTripTime if ($pingResult.Status -eq [System.Net.NetworkInformation.IPStatus]::Success) { Write-AzsReadinessLog -Message "IP Address '$ipAddress' responds to ping" -Type 'Warning' $result = 'Fail' } if ($count % 100 -eq 0) { Write-AzsReadinessLog -Message "Probed $count out of $($addressList.Count) IP addresses" -ToScreen } } $neighbors = Get-NetNeighbor | Where-Object {$_.State -eq 'Reachable'} | Select-Object -Property $netNeighborProperties $outputObject = @{'PingResults' = $pingOutput; 'NetNeighbors' = $neighbors} $ipRoutes = Get-NetRoute -AddressFamily IPv4 | Where-Object {$_.NextHop -ne '0.0.0.0'} $gatewayNeighbor = $neighbors | Where-Object {$_.IPAddress -in $ipRoutes.NextHop} foreach ($neighbor in $neighbors) { if ($neighbor.IPAddress -in $addressList) { if ($neighbor.LinkLayerAddress -in $gatewayNeighbor.LinkLayerAddress) { Write-AzsReadinessLog -Message "IP Address '$($neighbor.ipAddress)' resolves to the gateway MAC address. Proxy ARP is enabled on the network. Check that the IP address is not in use." -Type 'Warning' $failureDetail = 'Some IP addresses allocated to Azure Stack may be active on the network. Check the output log for more details.' if ($result -eq 'OK') { $result = 'WARNING' } } else { Write-AzsReadinessLog -Message "IP Address '$($neighbor.ipAddress)' is active on the local network" -Type 'Warning' $result = 'Fail' } } } if ($result -eq 'Fail') { $failureDetail = 'Some IP addresses allocated to Azure Stack are active on the network' Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies that the BGP peering has established. .DESCRIPTION Communicates with the TOR over SSH to determine if the BGP peering has successfully established. #> function Test-BgpPeering { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [SSH.SshSession] $SSHSession, [Parameter(Mandatory = $true)] [System.Management.Automation.PSCredential] $Credential, [Parameter()] [System.Int32] $Timeout = 60 ) $test = 'BGP Peering' Write-AzsReadinessLog -Message "Starting test '$test'" try { $result = 'Fail' $stopwatch = New-Object -TypeName System.Diagnostics.Stopwatch $stopwatch.Start() do { Write-AzsReadinessLog -Message "Retrieving BGP peering status from the TOR" $bgpStatus = Invoke-VirtualRouterCommand -SSHSession $SSHSession -Credential $Credential -Command 'show ip bgp summary' $peerStatusArray = $bgpStatus[11] -split ' ' | Where-Object {$_} $peerStatusString = $peerStatusArray[8] if ($peerStatusString) { Write-AzsReadinessLog -Message "Retrieved peer uptime status from the TOR: '$peerStatusString'" if ($peerStatusString -ne 'never') { Write-AzsReadinessLog -Message "BGP peer connection established" $result = 'OK' } else { Start-Sleep -Seconds 5 } } else { Write-AzsReadinessLog -Message "BGP status query did not return the expected output. Inspect the log for more information." -Type 'Warning' Start-Sleep -Seconds 5 } } until (($result -eq 'OK') -or ($stopwatch.Elapsed.TotalSeconds -ge $Timeout)) } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $peerStatusString} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies that the BGP peer is advertising the default route 0.0.0.0/0. .DESCRIPTION Communicates with the TOR over SSH to determine if the BGP default route is advertised. #> function Test-BgpDefaultRoute { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [SSH.SshSession] $SSHSession, [Parameter(Mandatory = $true)] [System.Management.Automation.PSCredential] $Credential, [Parameter()] [System.Int32] $Timeout = 60 ) $test = 'BGP Default Route' Write-AzsReadinessLog -Message "Starting test '$test'" try { $result = 'Fail' $stopwatch = New-Object -TypeName System.Diagnostics.Stopwatch $stopwatch.Start() do { Write-AzsReadinessLog -Message "Retrieving BGP routes from the TOR" $bgpNetwork = Invoke-VirtualRouterCommand -SSHSession $SSHSession -Credential $Credential -Command 'show ip bgp network' $bgpRoutesString = $bgpNetwork | Where-Object {$_ -like '?>*' -or $_ -like '?=*'} $bgpRoutes = @() foreach ($entry in $bgpRoutesString) { $entryData = $entry -split ' ' | Where-Object {$_} $bgpRoutes += [PSCustomObject]@{ Network = $entryData[1] NextHop = $entryData[2] Metric = $entryData[3] Weight = $entryData[4] Path = $entryData[5] } } if ($bgpRoutes | Where-Object {$_.Network -eq '0.0.0.0/0'}) { Write-AzsReadinessLog -Message "Default route is advertised" $result = 'OK' } else { Start-Sleep -Seconds 5 } } until (($result -eq 'OK') -or ($stopwatch.Elapsed.TotalSeconds -ge $Timeout)) } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Test failed with error: $failureDetail" -Type 'Error' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $bgpRoutes} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies DNS server connectivity and name resolution. .DESCRIPTION Checks that each DNS server can be reached over ICMP. Checks that each DNS server can be reached over TCP port 53. Tries to resolve a DNS name. #> function Test-DnsServer { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $DnsServer, [Parameter(Mandatory = $true)] [System.String] $ResolveName ) $test = "DNS Server $DnsServer" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' try { Write-AzsReadinessLog -Message "Testing DNS server '$DnsServer'" $tnc = Test-NetConnectionEx -RemoteAddress $DnsServer -Port 53 $dnsTest = Resolve-DnsName -Server $DnsServer -Name $ResolveName -DnsOnly Write-AzsReadinessLog -Message "Name '$ResolveName' resolved to IP addresses $($dnsTest.IPAddress -join ', ') using server $DnsServer" } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error resolving DNS name: '$failureDetail'" -Type 'Error' } finally { $outputObject = @{'NetConnection' = $tnc; 'DNSTest' = $dnsTest} } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies connectivity to destination using the maximum packet size. .DESCRIPTION Checks that the destination can be reached over ICMP. Discovers the network path MTU and compares with the host network configuration. #> function Test-PathMtu { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $Destination ) $test = "Network path MTU" Write-AzsReadinessLog -Message "Starting test '$test'" try { Write-AzsReadinessLog -Message "Checking the host network MTU setting" $ipRoutes = Get-NetRoute -AddressFamily IPv4 | Where-Object {$_.NextHop -ne '0.0.0.0'} $routedInterfaces = Get-NetIPInterface -AddressFamily IPv4 | Where-Object {$_.ConnectionState -eq 'Connected' -and $_.InterfaceIndex -in $ipRoutes.InterfaceIndex} $hostMtu = ($routedInterfaces | Sort-Object -Property NlMtu | Select-Object -Last 1).NlMtu Write-AzsReadinessLog -Message "Host MTU setting is $hostMtu bytes" $pathMtu = Invoke-PathMtuDiscovery -Destination $Destination -Mtu $hostMtu if ($pathMtu -eq $hostMtu) { Write-AzsReadinessLog -Message "Network path MTU to $Destination is greater or equal to the host setting of $hostMtu bytes" $result = 'OK' } elseif ($pathMtu -eq 0) { $failureDetail = "Destination host $Destination is unreachable. Unable to use it for path MTU discovery." Write-AzsReadinessLog -Message $failureDetail -Type 'Error' $result = 'Fail' } else { $failureDetail = "Discovered network path MTU is $pathMtu bytes. Configure the host network interface MTU to $pathMtu or less or increase the network infrastructure MTU to prevent packet loss." Write-AzsReadinessLog -Message $failureDetail -Type 'Error' $result = 'Fail' } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Unable to discover network path MTU: '$failureDetail'" -Type 'Error' } finally { $outputObject = @{'MTU' = $pathMtu} } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies time server connectivity and probes for current time. .DESCRIPTION Checks that each time server can be reached over ICMP. Tries to obtain time from the time server. #> function Test-TimeServer { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $TimeServer ) $test = "Time Server $TimeServer" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' Write-AzsReadinessLog -Message "Testing time server '$TimeServer'" try { $tnc = Test-NetConnectionEx -RemoteAddress $TimeServer Write-AzsReadinessLog -Message "Trying to get NTP time from '$TimeServer'" $stripchart = & w32tm.exe /stripchart /computer:$TimeServer /dataonly /samples:1 if ($stripchart.Count -ge 4 -and (($stripchart[3] -like '*, +*') -or ($stripchart[3] -like '*, -*'))) { Write-AzsReadinessLog -Message "Response received, current time and local offset: $($stripchart[3])" } else { $result = 'Fail' $failureDetail = $stripchart | Select-String -Pattern 'error' Write-AzsReadinessLog -Message "Did not receive an expected response from time server '$TimeServer'. $stripchart" -Type 'Error' } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error occurred: '$failureDetail'" -Type 'Error' } finally { $outputObject = @{'Stripchart' = $stripchart; 'NetConnection' = $tnc} } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies syslog server connectivity. .DESCRIPTION Checks that the syslog server can be reached over ICMP. Checks that the syslog server can be reached over TCP port 514. #> function Test-SyslogServer { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $SyslogServer ) $test = "Syslog Server $SyslogServer" Write-AzsReadinessLog -Message "Starting test '$test'" try { Write-AzsReadinessLog -Message "Testing Syslog server '$SyslogServer'" $tnc = Test-NetConnectionEx -RemoteAddress $SyslogServer -Port 514 if ($tnc.PingSucceeded -and $tnc.TcpTestSucceeded) { Write-AzsReadinessLog -Message "Syslog server $SyslogServer is reachable on TCP port 514" $result = 'OK' } else { Write-AzsReadinessLog -Message "Syslog server $SyslogServer ping result is '$($tnc.PingSucceeded)' and TCP test result is '$($tnc.TcpTestSucceeded)'." -Type 'Error' $result = 'Fail' } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error while testing connection: '$failureDetail'" -Type 'Error' } finally { $outputObject = $tnc } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies proxy server connectivity and authentication. .DESCRIPTION Attempts to make a web request to the specified proxy server. #> function Test-ProxyServer { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.Uri] $Proxy, [Parameter()] [System.Management.Automation.PSCredential] $ProxyCredential, [Parameter(Mandatory = $true)] [System.Uri] $TestUri ) $test = "Proxy Server $($Proxy.Host)" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' try { Write-AzsReadinessLog -Message "Testing network connection to the proxy server '$Proxy'" $tnc = Test-NetConnectionEx -RemoteAddress $Proxy.Host -Port $Proxy.Port # Web request to the proxy URL is expected to return 400/401/403 error with details in ErrorDetail. # But if ErrorDetails is empty, the proxy couldn't be resolved or didn't respond, so we will throw to the outer catch try { Write-AzsReadinessLog -Message "Testing HTTP connection to the proxy server '$Proxy'" $webResponse = Invoke-WebRequest -Uri $Proxy -UseBasicParsing -TimeoutSec 15 $proxyOut = $webResponse.BaseResponse } catch { if ($_.ErrorDetails) { $proxyOut = @{'Exception' = $_.Exception.Message; 'Details' = $_.ErrorDetails.Message} } else { throw $_ } } if (-not $ProxyCredential) { Write-AzsReadinessLog -Message "Attempting to access URI '$TestUri' with proxy server '$Proxy'" } else { Write-AzsReadinessLog -Message "Attempting to access URI '$TestUri' with proxy server '$Proxy' and username '$($ProxyCredential.UserName)'" } $webResponse = Invoke-WebRequest -Uri $TestUri @webParams $testOut = $webResponse.BaseResponse } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error while trying to use the proxy server '$Proxy': '$failureDetail'" -Type 'Error' } finally { $outputObject = @{'NetConnection' = $tnc; 'ProxyOutput' = $proxyOut; 'TestUrlOutput' = $testOut} } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies connectivity to Azure AD Login, Graph, Management Service, and Azure Resource Manager URIs .DESCRIPTION Attempts to make a web request to the Azure endpoints. #> function Test-AzureEndpoint { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $AzureEnvironment, [Parameter()] [System.Uri] $CustomCloudArmEndpoint, [Parameter(Mandatory = $true)] [ValidateSet( 'ARM', 'Graph', 'Login', 'ManagementService', 'DnsLoadBalancer', 'AseService', 'AseServiceBus', 'AseStorageAccount' )] [System.String] $EndpointType ) $test = "Azure $EndpointType Endpoint" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' $armEndpoint = switch ($AzureEnvironment) { 'AzureCloud' {'https://management.azure.com'} 'AzureUSGovernment' {'https://management.usgovcloudapi.net'} 'AzureChinaCloud' {'https://management.chinacloudapi.cn'} 'AzureGermanCloud' {'https://management.microsoftazure.de'} default {$CustomCloudArmEndpoint} } $graphEndpoint = switch ($AzureEnvironment) { 'AzureCloud' {'https://graph.windows.net'} 'AzureUSGovernment' {'https://graph.windows.net'} 'AzureChinaCloud' {'https://graph.chinacloudapi.cn'} 'AzureGermanCloud' {'https://graph.cloudapi.de'} } $loginEndpoint = switch ($AzureEnvironment) { 'AzureCloud' {'https://login.microsoftonline.com'} 'AzureUSGovernment' {'https://login.microsoftonline.us'} 'AzureChinaCloud' {'https://login.chinacloudapi.cn'} 'AzureGermanCloud' {'https://login.microsoftonline.de'} 'AzureUSSec' {'https://login.microsoftonline.microsoft.scloud'} 'AzureUSNat' {'https://login.microsoftonline.eaglex.ic.gov'} } $managementServiceEndpoint = switch ($AzureEnvironment) { 'AzureCloud' {'https://management.core.windows.net'} 'AzureUSGovernment' {'https://management.core.usgovcloudapi.net'} 'AzureChinaCloud' {'https://management.core.chinacloudapi.cn'} 'AzureGermanCloud' {'https://management.core.cloudapi.de'} } $trafficManagerEndpoint = switch ($AzureEnvironment) { 'AzureCloud' {'https://azstrpprod.trafficmanager.net'} } $aseServiceEndpoint = switch ($AzureEnvironment) { 'AzureCloud' { @( 'https://pod01-edg1.eus.databoxedge.azure.com' 'https://pod01-edg1.wus2.databoxedge.azure.com' 'https://pod01-edg1.sea.databoxedge.azure.com' 'https://pod01-edg1.we.databoxedge.azure.com' ) } 'AzureUSGovernment' {'https://pod01-edg1.ugv.databoxedge.azure.us'} 'AzureUSSec' {'https://pod01-edg1.euss.databoxedge.microsoft.scloud'} 'AzureUSNat' {'https://pod01-edg1.eusn.databoxedge.azure.eaglex.ic.gov'} } $aseServiceBusEndpoint = switch ($AzureEnvironment) { 'AzureCloud' { @( 'https://euspod01edg1sbcnpu53n.servicebus.windows.net' 'https://wus2pod01edg1sbcnqh26z.servicebus.windows.net' 'https://seapod01edg1sbcnkw22o.servicebus.windows.net' 'https://wepod01edg1sbcnhk23j.servicebus.windows.net' ) } 'AzureUSGovernment' {'https://ugvpod01edg1sbcnhx7pv.servicebus.usgovcloudapi.net'} 'AzureUSSec' {'https://eusspod01edg1sbcnlut32.servicebus.cloudapi.microsoft.scloud'} 'AzureUSNat' {'https://eusnpod01edg1sbcng19pt.servicebus.cloudapi.eaglex.ic.gov'} } $aseStorageAccountEndpoint = switch ($AzureEnvironment) { 'AzureCloud' { @( 'https://seapod1edg1monsa01kw22o.table.core.windows.net' 'https://seapod1edg1monsa02kw22o.table.core.windows.net' 'https://seapod1edg1monsa03kw22o.table.core.windows.net' 'https://euspod01edg1monsa01pu53n.table.core.windows.net' 'https://euspod01edg1monsa02pu53n.table.core.windows.net' 'https://euspod01edg1monsa03pu53n.table.core.windows.net' 'https://wus2pod1edg1monsa01qh26z.table.core.windows.net' 'https://wus2pod1edg1monsa02qh26z.table.core.windows.net' 'https://wus2pod1edg1monsa03qh26z.table.core.windows.net' 'https://wepod01edg1monsa01hk23j.table.core.windows.net' 'https://wepod01edg1monsa02hk23j.table.core.windows.net' 'https://wepod01edg1monsa03hk23j.table.core.windows.net' ) } 'AzureUSGovernment' { @( 'https://ugvpod1edg1monsa01hx7pv.table.core.usgovcloudapi.net' 'https://ugvpod1edg1monsa02hx7pv.table.core.usgovcloudapi.net' 'https://ugvpod1edg1monsa03hx7pv.table.core.usgovcloudapi.net' ) } 'AzureUSSec' { @( 'https://lut32eusspod1edg1monsa01.table.core.microsoft.scloud' 'https://lut32eusspod1edg1monsa02.table.core.microsoft.scloud' 'https://lut32eusspod1edg1monsa03.table.core.microsoft.scloud' ) } 'AzureUSNat' { @( 'https://g19pteusnpod1edg1monsa01.table.core.eaglex.ic.gov' 'https://g19pteusnpod1edg1monsa02.table.core.eaglex.ic.gov' 'https://g19pteusnpod1edg1monsa03.table.core.eaglex.ic.gov' ) } } $armUri = $armEndpoint.TrimEnd('/') + '/metadata/endpoints?api-version=2015-01-01' try { if ($EndpointType -eq 'ARM') { Write-AzsReadinessLog -Message "Testing ARM endpoint $armUri" $web = Invoke-WebRequestEx -Uri $armUri $outputObject = @{URI = $armUri; Output = $web} if ($web.WebResponse.StatusCode -eq 200) { Write-AzsReadinessLog -Message "Connection to ARM endpoint successful" } else { throw "Unable to retrieve Azure endpoints from ARM URI at $armEndpoint. Exception: $($web.Exception.ExceptionMessage)." } } else { if ($EndpointType -in 'Graph', 'Login', 'ManagementService' -and $CustomCloudArmEndpoint) { Write-AzsReadinessLog -Message "Invoking REST method to retrieve endpoints from '$armUri'" $azureEndpoints = Invoke-RestMethod -Uri $armUri @webParams $endpoint = switch ($EndpointType) { 'Graph' {$azureEndpoints.graphEndpoint} 'Login' {$azureEndpoints.authentication.loginEndpoint} 'ManagementService' {$azureEndpoints.authentication.audiences[0]} } } else { $endpoint = switch ($EndpointType) { 'Graph' {$graphEndpoint} 'Login' {$loginEndpoint} 'ManagementService' {$managementServiceEndpoint} 'DnsLoadBalancer' {$trafficManagerEndpoint} 'AseService' {$aseServiceEndpoint} 'AseServiceBus' {$aseServiceBusEndpoint} 'AseStorageAccount' {$aseStorageAccountEndpoint} } } if ($endpoint) { $outputObject = @() foreach ($uri in $endpoint) { $web = Invoke-WebRequestEx -Uri $uri $outputObject += @{URI = $uri; Output = $web} if ($web.Exception.NonHTTPFailure) { $result = 'Fail' $failureDetail = "Azure $EndpointType endpoint did not respond. Error: $($web.Exception.ExceptionMessage)" Write-AzsReadinessLog -Message $failureDetail -Type 'Error' } } } elseif ($EndpointType -in 'Graph', 'Login', 'ManagementService') { throw "Unable to retrieve Azure $EndpointType endpoint URL from '$armEndpoint'" } else { Write-AzsReadinessLog -Message "Endpoint $EndpointType for Azure environment $AzureEnvironment is not defined. The test will be skipped." $result = 'Skipped' } } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error while connecting to Azure endpoint: '$failureDetail'" -Type 'Error' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies connectivity to the ADFS server .DESCRIPTION Reads the federation metadata from the ADFS server specified in deployment data #> function Test-AdfsEndpoint { [CmdletBinding()] param ( [Parameter()] [System.Uri] $MetadataUri ) $test = "ADFS Metadata Endpoint" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' $failureDetail = @() try { $sslSetting = [Net.ServicePointManager]::SecurityProtocol Write-AzsReadinessLog -Message 'Forcing TLS 1.2 security' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 Write-AzsReadinessLog -Message "Retrieving ADFS metadata from '$MetadataUri'" $web = Invoke-WebRequestEx -Uri $MetadataUri -NoProxy if ($web.WebResponse.StatusCode -eq 200) { Write-AzsReadinessLog -Message "Successfully downloaded the metdata file" } else { throw "Unable to retrieve ADFS metadata. Exception: $($web.Exception.ExceptionMessage)." } Write-AzsReadinessLog -Message "Parsing ADFS metadata" [System.Xml.XmlDocument]$xml = $web.Content $outputObject = $xml if (-not $xml.EntityDescriptor) { $failureDetail += 'Entity Descriptor missing from federation metadata' } if (-not $xml.EntityDescriptor.EntityId) { $failureDetail += 'Entity Id in Entity Descriptor missing from federation metadata' } if (-not $xml.EntityDescriptor.RoleDescriptor) { $failureDetail += 'Role Descriptor missing from federation metadata' } if (-not $xml.EntityDescriptor.Signature) { $failureDetail += 'No signature in federation metadata file' } if ($failureDetail) { $result = 'Fail' } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error during ADFS endpoint validation: '$failureDetail'" -Type 'Error' } finally { Write-AzsReadinessLog -Message 'Restoring previous SSL settings' [Net.ServicePointManager]::SecurityProtocol = $sslSetting } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies connectivity to the LDAP and Global Catalog services required for Graph integration .DESCRIPTION Discovers the domain controller with DCLocator and runs the TCP port test #> function Test-Graph { [CmdletBinding()] param ( [Parameter()] [System.String] $ForestFqdn ) $test = "Active Directory Graph" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' $failureDetail = @() try { Write-AzsReadinessLog -Message "Testing network connection to the domain $ForestFqdn" $forestRootNet = Test-NetConnectionEx -RemoteAddress $ForestFqdn Write-AzsReadinessLog -Message "Attempting to discover the nearest domain controller running Windows Server 2012 or later using DC Locator" $dc = Get-ADDomainController -DomainName $ForestFqdn -NextClosestSite -Discover -ForceDiscover -MinimumDirectoryServiceVersion Windows2012 if ($dc) { Write-AzsReadinessLog -Message "Found the domain controller '$($dc.Hostname)' in site '$($dc.Site)' with the IP address '$($dc.IPv4Address)'." } else { throw "Unable to find a domain controller for $ForestFqdn" } Write-AzsReadinessLog -Message "Attempting to discover the global catalog server" $gc = Get-ADDomainController -DomainName $ForestFqdn -NextClosestSite -Discover -ForceDiscover -MinimumDirectoryServiceVersion Windows2012 -Service GlobalCatalog if ($gc) { Write-AzsReadinessLog -Message "Found the global catalog server '$($gc.Hostname)' in site '$($gc.Site)' with the IP address '$($gc.IPv4Address)'." } else { throw "Unable to find a global catalog server for $ForestFqdn" } $outputObject = @{ ForestRoot = $forestRootNet DomainController = $dc GlobalCatalog = $gc } $tests = @( [PSCustomObject]@{ Protocol = 'LDAP' Hostname = $dc.Hostname Port = 389 } [PSCustomObject]@{ Protocol = 'LDAPSSL' Hostname = $dc.Hostname Port = 636 } [PSCustomObject]@{ Protocol = "GC" Hostname = $gc.Hostname Port = 3268 } [PSCustomObject]@{ Protocol = 'GCSSL' Hostname = $gc.Hostname Port = 3269 } ) foreach ($netTest in $tests) { Write-AzsReadinessLog -Message "Testing network connectivity to '$($netTest.Hostname)' using $($netTest.Protocol) protocol (TCP port $($netTest.Port))" $tnc = Test-NetConnectionEx -RemoteAddress $netTest.Hostname -Port $netTest.Port $outputObject.($netTest.Protocol) = $tnc if (-not $tnc.TcpTestSucceeded) { $message = "$($netTest.Protocol) connection to $($netTest.Hostname) failed" Write-AzsReadinessLog -Message $message -Type 'Error' $failureDetail += $message $result = 'Fail' } } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error during Graph validation: '$failureDetail'" -Type 'Error' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies connectivity to Windows Update Service server URLs .DESCRIPTION Attempts to make a web request to the list of Windows Update or WSUS URLs. #> function Test-WindowsUpdateServer { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.Uri] $ServerUri ) $test = "Windows Update Server $($ServerUri.Host) port $($ServerUri.Port)" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' try { Write-AzsReadinessLog -Message "Attempting to access URI '$ServerUri'" if ($ServerUri.AbsoluteUri.EndsWith('.microsoft.com/') -or $ServerUri.AbsoluteUri.EndsWith('.windowsupdate.com/')) { $web = Invoke-WebRequestEx -Uri $ServerUri } else { $web = Invoke-WebRequestEx -Uri $ServerUri -NoProxy } $outputObject = $web if ($web.Exception.NonHTTPFailure) { throw "Unable to connect to URI $ServerUri. Exception: $($web.Exception.ExceptionMessage)." } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error while connecting to Windows Update Server: $failureDetail" -Type 'Error' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies connectivity to any user-specified URL .DESCRIPTION Attempts to make a web request to the provided URL. #> function Test-CustomUrl { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.Uri] $ServerUri ) $test = "URL $ServerUri" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' try { Write-AzsReadinessLog -Message "Attempting to access URI '$ServerUri'" $web = Invoke-WebRequestEx -Uri $ServerUri $outputObject = $web if ($web.Exception.NonHTTPFailure) { throw "Unable to connect to URI $ServerUri. Exception: $($web.Exception.ExceptionMessage)." } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error while connecting to server: $failureDetail" -Type 'Error' } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies that the DNS resource record is present. .DESCRIPTION Checks that the DNS server contains specific resource records required by Azure Stack Edge. #> function Test-DnsRegistration { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String[]] $DnsServer, [Parameter(Mandatory = $true)] [System.String] $DnsRecord ) $test = "DNS Registration for $DnsRecord" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' try { $DnsRecord = $DnsRecord.Replace('*', 'testname') Write-AzsReadinessLog -Message "Using DNS servers '$($DnsServer -join ', ')' to resolve name '$DnsRecord'" $dnsTest = Resolve-DnsName -Server $DnsServer -Name $DnsRecord -DnsOnly Write-AzsReadinessLog -Message "Name '$DnsRecord' resolved to IP addresses $($dnsTest.IPAddress -join ', ')" } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error resolving DNS name: '$failureDetail'" -Type 'Error' } finally { $outputObject = $dnsTest } Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Verifies that the DNS domain delegation is configured on the customer's DNS server. .DESCRIPTION Installs a DNS server, creates a resource record, and attempts to resolve the resource record using the customer's DNS server. The test will only succeed if the customer-provided DNS server has been configured to forward DNS requests to the Azure Stack Hub DNS server. #> function Test-DnsDelegation { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $DnsDomain, [Parameter(Mandatory = $true)] [System.Net.IPAddress] $IPv4Address ) $test = "DNS Delegation" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' try { Write-AzsReadinessLog -Message "Installing DNS Server on the local machine" Install-DnsServer -ZoneName $DnsDomain -RecordName 'nrc' -IPv4Address $IPv4Address $fqdn = "nrc.$DnsDomain" Write-AzsReadinessLog -Message "Attempting to resolve the test name '$fqdn'" $dnsTest = Resolve-DnsName -Name $fqdn -DnsOnly Write-AzsReadinessLog -Message "Name '$fqdn' resolved to IP address $($dnsTest.IPAddress)" if ($dnsTest.IPAddress -ne $IPv4Address) { $failureDetail = "The resolved IP address '$($dnsTest.IPAddress)' does not match the expected IP address '$IPv4Address'. Check your DNS Delegation configuration." Write-AzsReadinessLog -Message $failureDetail -Type 'Error' } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error testing DNS delegation: '$failureDetail'" -Type 'Error' } finally { Write-AzsReadinessLog -Message "Removing DNS Server from the local machine" Remove-DnsServer -Uninstall } $outputObject = $dnsTest Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Runs the network readiness tests on the HLH using the DVM IP address. .DESCRIPTION Used in the HLH mode. #> function Test-BmcNetworkOnHlh { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $DeploymentDataPath, [Parameter(Mandatory = $true)] [System.String] $VirtualSwitchName, [Parameter()] [System.Boolean] $NoUplinksRequired, [Parameter()] [System.Uri] $CustomCloudArmEndpoint ) $test = "BMC Management Network (DVM) Ready for Deployment" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' try { $networkTesterPath = Join-Path -Path $PSScriptRoot -ChildPath 'NetworkChecker\networktester.ps1' Write-AzsReadinessLog -Message "Calling $networkTesterPath" $bmcTest = & $networkTesterPath -DeploymentDataJson $DeploymentDataPath -VirtualSwitchName $VirtualSwitchName -NoUplinksRequired $NoUplinksRequired -CustomCloudArmEndpoint $CustomCloudArmEndpoint if (-not $bmcTest.Success) { $result = 'Fail' $failureDetail = "BMC Management Network is not ready for deployment. See log file for traceroute information." } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error verifying the BMC network: '$failureDetail'" -Type 'Error' } $outputObject = $bmcTest Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Runs the network readiness tests on the HLH using the SONiC VM. .DESCRIPTION Used in the HLH mode. The SONiC VM must be up and running with the TestData.json file uploaded to it. #> function Test-ExternalNetworkOnHlh { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [SSH.SshSession] $VirtualRouterSession, [Parameter(Mandatory = $true)] [System.String] $CloudID, [Parameter(Mandatory = $true)] [System.Management.Automation.PSCredential] $Credential, [Parameter(Mandatory = $true)] [System.String] $LogDestinationPath ) $test = "External Network Ready for Deployment" Write-AzsReadinessLog -Message "Starting test '$test'" $result = 'OK' try { # Create temporary folder for result files from SONiC VM $tmpDirectory = Join-Path -Path $env:TEMP -ChildPath (New-Guid) Write-AzsReadinessLog -Message "Using temporary folder $tmpDirectory" $null = New-Item -Path $tmpDirectory -Itemtype Directory # Execute the External Network Tests in SONiC. Max timeout for 2 tors is 30 minutes. $null = Invoke-VirtualRouterCommand -Command "sudo python3 Microsoft.AzureStack.ReadinessChecker.NetworkValidation.py $cloudID" -SSHSession $VirtualRouterSession -Credential $Credential -Timeout 1800 -SuccessCodes 0, 2 # Get results.json, nrc.log, and traceroutes. Get-SCPItem -ComputerName $VirtualRouterSession.Host -Credential $Credential -AcceptKey:$true -Path "./results/$CloudID/" -PathType Directory -Destination $tmpDirectory $resultsFiles = Get-ChildItem -Path $tmpDirectory -Recurse foreach ($item in $resultsFiles | Where-Object {$_.Name -match '^results.json$|^nrc.log$|^traceroute$'}) { Copy-Item -Path $item.FullName -Destination $LogDestinationPath -Recurse -Force if ($item.Name -eq 'results.json') { $externalResults = Get-Content -Path $item.FullName | ConvertFrom-Json } if ($item.FullName -match '.*\\traceroute\\tor.*') { $troute = Get-Content -Path $item.FullName $trouteName = $item.Name -replace '.txt' Write-AzsReadinessLog -Message "Traceroute for: $trouteName" $troute | ForEach-Object {Write-AzsReadinessLog -Message $_} } } if (-not $externalResults.Success) { $result = 'Fail' $failureDetail = $externalResults.Message } } catch { $result = 'Fail' $failureDetail = $_.Exception.Message Write-AzsReadinessLog -Message "Error verifying the External network: '$failureDetail'" -Type 'Error' } finally { if (Test-Path -Path $tmpDirectory) { Write-AzsReadinessLog -Message "Removing temporary folder $tmpDirectory" Remove-Item -Path $tmpDirectory -Recurse -Force } } $outputObject = $externalResults Write-AzsReadinessLog -Message "Test '$test' finished, result '$result'" $hash = @{'Test' = $test; 'Result' = $result; 'FailureDetail' = $failureDetail; 'OutputObject' = $outputObject} $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Wrapper function for Test-Connection. .DESCRIPTION Pings a remote computer for a number of times, returning immediately after the first successful ping. #> function Test-ConnectionEx { [CmdletBinding()] [OutputType([System.Boolean])] param ( [Parameter(Mandatory = $true)] [System.String] $RemoteAddress, [Parameter()] [System.Int32] $Count ) for ($i = 0; $i -le $Count; $i++) { Write-AzsReadinessLog -Message "Pinging $RemoteAddress" if (Test-Connection -ComputerName $RemoteAddress -Count 1 -Quiet) { return $true } } return $false } <# .SYNOPSIS Wrapper function for Test-NetConnection. For internal use. .DESCRIPTION Tests network connection with trace route and TCP test. Writes to readiness log and returns a PSObject with results. #> function Test-NetConnectionEx { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $RemoteAddress, [Parameter()] [System.Int32] $Port ) $ipAddress = $null $silentlyContinue = [System.Management.Automation.ActionPreference]::SilentlyContinue if ([System.Net.IPAddress]::TryParse($RemoteAddress, [ref]$ipAddress)) { Write-AzsReadinessLog -Message "'$RemoteAddress' is a valid IP address" } else { Write-AzsReadinessLog -Message "'$RemoteAddress' is a DNS name, attempting to resolve" $dnsName = Resolve-DnsName -Server $DnsServer -Name $RemoteAddress -DnsOnly Write-AzsReadinessLog -Message "'$RemoteAddress' resolved to IP address '$($dnsName.IPAddress -join ', ')'" } Write-AzsReadinessLog -Message "Tracing route to '$RemoteAddress'" $traceRoute = Test-NetConnection -ComputerName $RemoteAddress -TraceRoute -Hops 10 -WarningAction $silentlyContinue if ($traceRoute.PingSucceeded) { Write-AzsReadinessLog -Message "ICMP test to '$RemoteAddress' succeeded using interface '$($traceRoute.InterfaceAlias)' with source address '$($traceRoute.SourceAddress.IPAddress)'. Route: $($traceRoute.TraceRoute -join ', ')." } else { Write-AzsReadinessLog -Message "ICMP test to '$RemoteAddress' failed using interface '$($traceRoute.InterfaceAlias)' with source address '$($traceRoute.SourceAddress.IPAddress)'. Route: $($traceRoute.TraceRoute -join ', ')." -Type 'Warning' } if ($Port) { Write-AzsReadinessLog -Message "Validating TCP connection to '$RemoteAddress' port $Port" $tcpTest = Test-NetConnection -ComputerName $RemoteAddress -Port $Port -WarningAction $silentlyContinue if ($tcpTest.TcpTestSucceeded) { Write-AzsReadinessLog -Message "TCP connection to '$RemoteAddress' port '$Port' succeeded. Reverse lookup record '$($tcpTest.DnsOnlyRecords.NameHost)'." } else { Write-AzsReadinessLog -Message "TCP connection to '$RemoteAddress' port '$Port' failed" -Type 'Warning' } } $hash = @{ 'ComputerName' = $traceRoute.ComputerName 'RemotePort' = $tcpTest.RemotePort 'InterfaceAlias' = $traceRoute.InterfaceAlias 'PingSucceeded' = $traceRoute.PingSucceeded 'TcpTestSucceeded' = $tcpTest.TcpTestSucceeded 'TraceRoute' = $traceRoute.TraceRoute } $object = New-Object -TypeName 'PSObject' -Property $hash return $object } <# .SYNOPSIS Wrapper function for Invoke-WebRequest. For internal use. .DESCRIPTION Tests network connection with trace route and TCP test, then makes a web request. Writes to readiness log and returns a PSObject with results. #> function Invoke-WebRequestEx { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.Uri] $Uri, [Parameter()] [switch] $NoProxy ) if (-not $NoProxy -and $webParams.Proxy) { Write-AzsReadinessLog -Message "Skipping direct connection test to $($Uri.Host) because proxy server was specified" $tnc = "Skipped" } else { Write-AzsReadinessLog -Message "Testing network connection to the host '$($Uri.Host)'" $tnc = Test-NetConnectionEx -RemoteAddress $Uri.Host -Port $Uri.Port } try { Write-AzsReadinessLog -Message "Attempting to access web URI '$uri'" if (-not $NoProxy -and $webParams.Proxy) { $webOut = Invoke-WebRequest -Uri $Uri @webParams } else { $webOut = Invoke-WebRequest -Uri $Uri -TimeoutSec 15 -UseBasicParsing } $webResponse = $webOut.BaseResponse $headers = $webOut.Headers $content = $webOut.Content } catch { $webResponse = $_.Exception.Response if ($webResponse) { try { $headers = @{} $content = [System.Text.Encoding]::UTF8.GetString($webResponse.GetResponseStream().ToArray()) foreach ($header in $webResponse.Headers) { $headers.$header = $webResponse.GetResponseHeader($header) } if ($webResponse.ContentType -eq 'application/json') { $content = ConvertFrom-Json -InputObject $content } } catch {} } $exception = @{ExceptionMessage = $_.Exception.Message; ErrorDetails = $_.ErrorDetails.Message; NonHTTPFailure = [System.String]::IsNullOrEmpty($webResponse)} } $object = New-Object -TypeName 'PSObject' -Property @{'NetConnection' = $tnc; 'WebResponse' = $webResponse; 'Headers' = $headers; 'Content' = $content; 'Exception' = $exception} return $object } <# .SYNOPSIS Parameter validation helper for URI type. For internal use. .DESCRIPTION Tests the parameter is an absolute URI of a supported protocol. #> function Test-Uri { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.Uri] $Uri, [Parameter()] [System.String[]] $SupportedProtocols = @('http', 'https') ) $result = $true Write-AzsReadinessLog -Message "Testing URI '$uri'" if (-not $Uri.IsAbsoluteUri) { Write-AzsReadinessLog -Message "Invalid URI value: '$Uri' is not a valid URI" -Type 'Error' -ToScreen $result = $false } elseif ($Uri.Scheme -notin $SupportedProtocols) { Write-AzsReadinessLog -Message "Invalid protocol for URI: '$($Uri.Scheme)'. Supported protocols are: $($SupportedProtocols -join ', ')." -Type 'Error' -ToScreen $result = $false } return $result } <# .SYNOPSIS Returns an IP address when provided a subnet address and an offset. For internal use. #> function Get-IPAddressOffset { [CmdletBinding()] [OutputType([System.Net.IPAddress])] param ( [Parameter(Mandatory = $true)] [System.Net.IPAddress] $SubnetAddress, [Parameter(Mandatory = $true)] [System.Int32] $Offset ) $ipInt = ([System.Net.IPAddress][System.String]([System.Net.IPAddress]$SubnetAddress).Address).Address $ipInt = $ipInt + $Offset $result = [System.Net.IPAddress][System.String]$ipInt return $result } <# .SYNOPSIS Prepare the NRC host for running Layer 3 tests. .DESCRIPTION Configure Hyper-V environment, provision SONiC virtual machines and apply initial configuration to simulate Azure Stack Hub physical network stack. #> function Start-VirtualRouter { [CmdletBinding()] param ( [Parameter()] [ValidatePattern("TOR|CL\d{2}-SonicMux")] [System.String] $VMName = 'TOR', # Full Deployment Data Json [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] [System.Object] $DeploymentData, [Parameter(Mandatory = $true)] [ValidateScript({Test-Path -Path $_ -Type Leaf})] [System.String] $ImagePath, [Parameter(Mandatory = $true)] [System.Management.Automation.PSCredential] $Credential, [Parameter()] [System.Int32] $UplinkCount, [Parameter(Mandatory = $false)] [System.String] $HlhVirtualSwitchName ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Write-AzsReadinessLog -Message 'Starting virtual network environment' # Parsing configuration $cloudID = Get-CloudID -DeploymentData $DeploymentData $dnsServer = $deploymentData.ScaleUnits.DeploymentData.DnsForwarder $mgmtNetworkName = 'Rack01-SwitchMgmt' $mgmtNetworkNameFe = "$cloudID-NRC-Mgmt" $mgmtNetwork = $DeploymentData.ConfigData.IPConfigData | Where-Object {$_.Name -in @($mgmtNetworkName, $mgmtNetworkNameFe)} | Select-Object -First 1 $mgmtSwitchName = $mgmtNetwork.Name $mgmtHostIP = $mgmtNetwork.IPv4Gateway $bmcSwitchName = 'Rack01-BMCMgmt' $bmcNetwork = $DeploymentData.ConfigData.IPConfigData | Where-Object {$_.Name -eq $bmcSwitchName} $bmcHostIp = $DeploymentData.ScaleUnits.DeploymentData.DeploymentVM.IPAddress $externalSwitchName = "$cloudID-External-VIPS" $externalNetwork = $DeploymentData.ConfigData.IPConfigData | Where-Object {$_.Name -eq $externalSwitchName} $externalTestIP = Get-IPAddressOffset -SubnetAddress $externalNetwork.IPv4NetworkAddress -Offset 15 if (-not ($mgmtHostIp -and $bmcHostIp -and $externalTestIP)) { Write-AzsReadinessLog -Message 'Error parsing configuration' -Type 'Error' return } try { # Configure host virtual interface for the control plane connection from the host to SONiC Write-AzsReadinessLog -Message "Management network name: $mgmtSwitchName" Write-AzsReadinessLog -Message "Adding host network adapter for SONiC management with IP address $mgmtHostIP" Assert-HostVNic -SwitchName $mgmtSwitchName -IPAddress $mgmtHostIP -PrefixLength $mgmtNetwork.IPv4MaskBits # Install and configure DHCP Server $scopes = @( @{ Name = $mgmtSwitchName ScopeId = $mgmtNetwork.IPv4NetworkAddress StartRange = $mgmtNetwork.IPv4FirstAddress EndRange = $mgmtNetwork.IPv4LastAddress SubnetMask = $mgmtNetwork.IPv4Mask } ) Install-DHCPServer -Scopes $scopes Write-AzsReadinessLog -Message "Configuring DHCP server binding" Get-DhcpServerv4Binding | Where-Object {$_.IPAddress -ne $mgmtHostIP} | Set-DhcpServerv4Binding -BindingState $false # Create SONiC virtual machine if (Get-VM | Where-Object {$_.Name -eq $VMName}) { Write-AzsReadinessLog -Message "Removing the existing virtual machine $vmName" Stop-VM -Name $VMName -TurnOff -Force Remove-VM -Name $VMName -Force } $imageParentPath = Split-Path -Path $ImagePath -Parent $imageFile = Get-Item -Path $ImagePath $diffFileName = $imageFile.BaseName + '.diff' + $imageFile.Extension $diffFilePath = Join-Path -Path $imageParentPath -ChildPath $diffFileName if (Test-Path -Path $diffFilePath) { Write-AzsReadinessLog -Message "Removing the image file $diffFilePath" Remove-Item -Path $diffFilePath -Force } Write-AzsReadinessLog -Message "Creating a new differencing image $diffFilePath" $null = New-VHD -Path $diffFilePath -ParentPath $ImagePath -Differencing Write-AzsReadinessLog -Message "Creating a new virtual machine $VMName and connecting the control plane vNIC to the virtual switch '$mgmtSwitchName'" $null = New-VM -Name $VMName -MemoryStartupBytes 3GB -Generation 1 -VHDPath $diffFilePath -SwitchName $mgmtSwitchName Set-VM -Name $VMName -ProcessorCount 4 # NRC HLH Mode uses a single data plane NIC Enternet0 connected to an existing Hyper-V external switch # - The HLH Mode is used when the UplinkCount parameter is not provided or set to 0 # NRC Appliance mode uses the following network layout: # - Ethernet0 - Internal Hyper-V switch simulating the BMC network # - Ethernet4 - Internal Hyper-V switch simulating the External-VIPS network # - Ethernet8 - External Hyper-V switch attached to the physical host NIC #1 # - Ethernet12 - External Hyper-V switch attached to the physical host NIC #2 (optional) # - Ethernet16 - External Hyper-V switch attached to the physical host NIC #3 (optional) # - Ethernet20 - External Hyper-V switch attached to the physical host NIC #4 (optional) if ($UplinkCount -eq 0) { Write-AzsReadinessLog -Message "Running NRC in the HLH mode using existing Hyper-V switch '$HlhVirtualSwitchName'" Add-VMNetworkAdapter -VMName $VMName -Name 'Ethernet0' Connect-VMNetworkAdapter -VMName $VMName -Name 'Ethernet0' -SwitchName $HlhVirtualSwitchName } else { Write-AzsReadinessLog -Message "Running NRC in the appliance mode - adding host network adapters for BMC and External networks" Assert-HostVNic -SwitchName $bmcSwitchName -IPAddress $bmcHostIP -PrefixLength $bmcNetwork.IPv4MaskBits -DefaultGateway $bmcNetwork.IPv4Gateway -DnsServer $DnsServer Assert-HostVNic -SwitchName $externalSwitchName -IPAddress $externalTestIP -PrefixLength $externalNetwork.IPv4MaskBits -DefaultGateway $externalNetwork.IPv4Gateway -DnsServer $DnsServer # The first two data plane vNICs are used for the BMC and External-VIPS vSwitch connections for ($vnicNumber = 0; $vnicNumber -lt $UplinkCount + 2; $vnicNumber++) { $ethNumber = $vnicNumber * 4 $vNicName = "Ethernet$ethNumber" Write-AzsReadinessLog -Message "Adding VM network adapter $vNicName" Add-VMNetworkAdapter -VMName $VMName -Name $vNicName } Connect-VMNetworkAdapter -VMName $VMName -Name 'Ethernet0' -SwitchName $bmcSwitchName Connect-VMNetworkAdapter -VMName $VMName -Name 'Ethernet4' -SwitchName $externalSwitchName } Get-VMNetworkAdapter -VMName $VMName -Name 'Ethernet*' | Set-VMNetworkAdapter -MacAddressSpoofing On Write-AzsReadinessLog -Message "Starting the virtual machine $VMName" Start-VM -Name $VMName # Wait for the VM to obtain a management IP address from DHCP $vmMgmtIP = $null $ssh = $null $retry = 0 do { Write-AzsReadinessLog -Message "Waiting 10 seconds before attempting discovery" Start-Sleep -Seconds 10 $retry++ try { $dhcpIpAddresses = $null Write-AzsReadinessLog -Message "Getting DHCP assigned IP addresses" $dhcpIpAddresses = (Get-DhcpServerv4Lease -ScopeId $mgmtNetwork.IPv4NetworkAddress).IPAddress.IPAddressToString } catch { Write-AzsReadinessLog -Message "Failed getting DHCP assigned addresses. Error: $($_.Exception.Message)" } foreach ($dhcpIpAddress in $dhcpIpAddresses) { Write-AzsReadinessLog -Message "Testing DHCP-assigned IP address $dhcpIpAddress" if (Test-Connection -ComputerName $dhcpIpAddress -Count 1 -Quiet) { $vmMgmtIP = $dhcpIpAddress Write-AzsReadinessLog -Message "Ping to the virtual router at $vmMgmtIP successful" try { $ssh = Assert-VirtualRouterSession -ComputerName $vmMgmtIP -Credential $Credential Write-AzsReadinessLog -Message "SSH connection to the virtual router $vmMgmtIP successful" } catch { Write-AzsReadinessLog -Message "Still waiting to establish SSH connection" Write-AzsReadinessLog -Message "Posh-SSH Exception: $($_.Exception.Message)" } } } } until ($ssh -or $retry -ge 30) if (-not $ssh) { Write-AzsReadinessLog -Message "Unable to establish SSH connection with the virtual router after $retry attempts" -Type 'Error' return } } catch { $file = $_.InvocationInfo.ScriptName $line = $_.InvocationInfo.ScriptLineNumber $exceptionMessage = $_.Exception.Message $message = "$file : $line >> $exceptionMessage" Write-AzsReadinessLog -Message "Start-VirtualRouter Failed: $message" -Type 'Error' -ToScreen return } Write-AzsReadinessLog -Message "Sleeping for 30 seconds before returning to allow SONiC containers to start" Start-Sleep -Seconds 30 return $ssh } <# .SYNOPSIS Creates or validates the Hyper-V Management OS virtual network adapter with IP configuration. .DESCRIPTION For internal use. #> function Assert-HostVNic { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $SwitchName, [Parameter(Mandatory = $true)] [System.Net.IPAddress] $IPAddress, [Parameter(Mandatory = $true)] [System.Int32] $PrefixLength, [Parameter()] [System.Net.IPAddress] $DefaultGateway, [Parameter()] [System.String[]] $DnsServer ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop # Create the virtual switch if (Get-VMSwitch | Where-Object {$_.Name -eq $SwitchName}) { Write-AzsReadinessLog -Message "Using the existing Hyper-V switch $SwitchName" } else { Write-AzsReadinessLog -Message "Creating new Hyper-V Switch $SwitchName" $null = New-VMSwitch -Name $SwitchName -SwitchType Internal } # Configure internal host vNIC $vNic = Get-VMNetworkAdapter -ManagementOS -SwitchName $SwitchName $nic = Get-NetAdapter | Where-Object {$_.DeviceId -eq $vNic.DeviceId} foreach ($currentIP in (Get-NetIPAddress | Where-Object {$_.AddressFamily -eq 'IPv4' -and $_.InterfaceIndex -eq $nic.InterfaceIndex})) { if ($currentIP.IPAddress -eq $IPAddress -and $currentIP.PrefixLength -eq $PrefixLength) { Write-AzsReadinessLog -Message "IP address $IPAddress is already assigned to the host interface $($nic.InterfaceAlias)" $ipAssigned = $true } elseif ($currentIP.PrefixOrigin -eq 'Manual') { Write-AzsReadinessLog -Message "Removing IP address $($currentIP.IPAddress) from the host interface $($nic.InterfaceAlias)" Remove-NetIPAddress -InterfaceIndex $nic.InterfaceIndex -IPAddress $currentIP.IPAddress -Confirm:$false Start-Sleep -Seconds 5 } } if (-not $ipAssigned) { Write-AzsReadinessLog -Message "Setting IP Address $IPAddress on the host interface $($nic.InterfaceAlias)" if ($DefaultGateway) { $null = New-NetIPAddress -InterfaceIndex $nic.InterfaceIndex -IPAddress $IPAddress -PrefixLength $PrefixLength -DefaultGateway $DefaultGateway } else { $null = New-NetIPAddress -InterfaceIndex $nic.InterfaceIndex -IPAddress $IPAddress -PrefixLength $PrefixLength } Start-Sleep -Seconds 5 } if ($DnsServer) { Write-AzsReadinessLog -Message "Setting DNS Servers $DnsServer on the host interface $($nic.InterfaceAlias)" Set-DnsClientServerAddress -InterfaceIndex $nic.InterfaceIndex -ServerAddresses $DnsServer } } <# .SYNOPSIS Clean up Layer 3 test artefacts on the NRC host. .DESCRIPTION Remove the SONiC virtual machines and Hyper-V objects. #> function Stop-VirtualRouter { [CmdletBinding()] param ( [Parameter()] [System.String] $VMName = 'TOR', [Parameter(Mandatory = $false)] [System.String] $CloudID = "CL01", [Parameter(Mandatory = $false)] [System.Boolean] $IsLabScenario = $false ) $switchNames = @('Rack01-SwitchMgmt', 'Rack01-BMCMgmt', "$cloudID-NRC-Mgmt") Write-AzsReadinessLog -Message 'Stopping virtual network environment' try { if ($vm = Get-VM | Where-Object {$_.Name -eq $VMName}) { Write-AzsReadinessLog -Message "Removing the existing virtual machine $VMName" $vhdPath = $vm.HardDrives.Path Stop-VM -Name $VMName -TurnOff -Force Remove-VM -Name $VMName -Force Remove-Item -Path $vhdPath -Force } foreach ($vmSwitch in (Get-VMSwitch | Where-Object {$_.Name -in $switchNames -or $_.Name -like 'TOR-Uplink-*'})) { Write-AzsReadinessLog -Message "Removing Hyper-V virtual switch '$($vmSwitch.Name)'" Remove-VMSwitch -Name $vmSwitch.Name -Force } # Only remove the service if this is a customer environment. Lab Scenario will keep the DHCP Service running. if ((Get-Service | Where-Object {$_.Name -eq 'DhcpServer'}) -and $IsLabScenario -eq $false) { Write-AzsReadinessLog -Message "Removing DHCP Server" Remove-DHCPServer -Uninstall } } catch { Write-AzsReadinessLog -Message "Error occured: $($_.Exception.Message)" -Type 'Error' return $false } return $true } <# .SYNOPSIS Installs and configures DHCP server to be used for NRC management network. .DESCRIPTION Used in virtual router scenarios. .PARAMETER Scopes Array of hashtables of scope parameters. #> function Install-DHCPServer { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.Collections.Hashtable[]] $Scopes ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop try { if (-not (Get-Service | Where-Object {$_.Name -eq 'DhcpServer'})) { Write-AzsReadinessLog -Message 'Installing DHCP server and management tools.' $null = Install-WindowsFeature -Name DHCP -IncludeManagementTools } if ((Get-Service -Name 'DhcpServer').Status -ne [System.ServiceProcess.ServiceControllerStatus]::Running) { Write-AzsReadinessLog -Message 'Starting DHCP server service.' Start-Service -Name DhcpServer -WarningAction SilentlyContinue } if (-not (Get-Module -Name DhcpServer)) { Write-AzsReadinessLog -Message 'Importing DHCP server module.' Import-Module -Name DhcpServer } Write-AzsReadinessLog -Message 'Configuring DHCP server settings' Set-DhcpServerSetting -ConflictDetectionAttempts 1 New-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Services\DHCPServer\Parameters\ -Name 'DisableRogueDetection' -Value 1 -ErrorAction SilentlyContinue Restart-Service -Name DhcpServer -WarningAction SilentlyContinue foreach ($scope in $Scopes) { if (-not (Get-DhcpServerv4Scope | Where-Object {$_.ScopeId -eq $scope.ScopeId})) { Write-AzsReadinessLog -Message "Creating DHCP server scope '$($scope.Name)' for network '$($scope.ScopeId)'." $scopeParams = @{ Name = $scope.Name StartRange = $scope.StartRange EndRange = $scope.EndRange SubnetMask = $scope.SubnetMask State = 'Active' LeaseDuration = (New-TimeSpan -Days 1) } Add-DhcpServerv4Scope @scopeParams } } } catch { $errorMessage = "Failed installing DHCP server. Error: $($_.Exception.Message)" Write-AzsReadinessLog -Message $errorMessage -Type 'Error' throw $errorMessage } } <# .SYNOPSIS Removes DHCP server. .DESCRIPTION Used in virtual router scenarios. #> function Remove-DHCPServer { [CmdletBinding()] [Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseShouldProcessForStateChangingFunctions", "")] param ( [Parameter()] [switch] $Uninstall ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop try { try { if (-not (Get-Module -Name DhcpServer)) { Write-AzsReadinessLog -Message 'Importing DHCP server module.' Import-Module -Name DhcpServer } } catch { Write-AzsReadinessLog -Message 'Failed importing DHCP server module.' -Type 'Error' } if (Get-Module -Name DhcpServer) { try { foreach ($scope in (Get-DhcpServerv4Scope)) { Write-AzsReadinessLog -Message "Removing scope id '$($scope.ScopeId)'." Remove-DhcpServerv4Scope -ScopeId $scope.ScopeId -Force } } catch { Write-AzsReadinessLog -Message "Failed removing DHCP server scopes. Error: $($_.Exception.Message)" -Type 'Error' } } if ( ($dhcpService = Get-Service | Where-Object {$_.Name -eq 'DhcpServer'}) -and ($dhcpService.Status -ne [System.ServiceProcess.ServiceControllerStatus]::Stopped) ) { Write-AzsReadinessLog -Message 'Stopping DHCP server service.' Stop-Service -Name 'DhcpServer' -WarningAction SilentlyContinue } if ($Uninstall) { $windowsFeatures = Get-WindowsFeature | Where-Object {$_.Installed} if ($windowsFeatures.Name -contains 'RSAT-DHCP') { Write-AzsReadinessLog -Message "Uninstalling DHCP management tools." $null = Uninstall-WindowsFeature -Name 'RSAT-DHCP' -WarningAction SilentlyContinue } if ($windowsFeatures.Name -contains 'DHCP') { Write-AzsReadinessLog -Message "Uninstalling DHCP server." $null = Uninstall-WindowsFeature -Name 'DHCP' -WarningAction SilentlyContinue } } } catch { $errorMessage = "Failed removing DHCP server. Error: $($_.Exception.Message)" Write-AzsReadinessLog -Message $errorMessage -Type 'Error' throw $errorMessage } } <# .SYNOPSIS Installs and configures DNS server to be used for the DNS Delegation test. .DESCRIPTION Used to verify that the customer DNS server is configured to forward DNS queries to Azure Stack. #> function Install-DnsServer { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $ZoneName, [Parameter()] [System.String] $RecordName = 'nrc', [Parameter()] [System.Net.IPAddress] $IPv4Address ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop try { if (-not (Get-Service | Where-Object {$_.Name -eq 'Dns'})) { Write-AzsReadinessLog -Message 'Installing DNS server and management tools.' $null = Install-WindowsFeature -Name DNS -IncludeManagementTools } if ((Get-Service -Name 'Dns').Status -ne [System.ServiceProcess.ServiceControllerStatus]::Running) { Write-AzsReadinessLog -Message 'Starting DNS server service.' Start-Service -Name Dns -WarningAction SilentlyContinue } if (-not (Get-Module -Name DnsServer)) { Write-AzsReadinessLog -Message 'Importing DNS server module.' Import-Module -Name DnsServer } Write-AzsReadinessLog -Message "Creating DNS zone $ZoneName" Add-DnsServerPrimaryZone -Name $ZoneName -ZoneFile "$ZoneName.dns" Write-AzsReadinessLog -Message "Crating A-Record $RecordName with the IP address $IPv4Address" Add-DnsServerResourceRecordA -ZoneName $ZoneName -Name $RecordName -IPv4Address $IPv4Address } catch { $errorMessage = "Failed installing DNS server. Error: $($_.Exception.Message)" Write-AzsReadinessLog -Message $errorMessage -Type 'Error' throw $errorMessage } } <# .SYNOPSIS Removes DNS server. .DESCRIPTION Used in the DNS Delegation test. #> function Remove-DnsServer { [CmdletBinding()] [Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseShouldProcessForStateChangingFunctions", "")] param ( [Parameter()] [switch] $Uninstall ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop try { try { if (-not (Get-Module -Name DnsServer)) { Write-AzsReadinessLog -Message 'Importing DNS server module.' Import-Module -Name DnsServer } } catch { Write-AzsReadinessLog -Message 'Failed importing DNS server module.' -Type 'Error' } if (Get-Module -Name DnsServer) { try { foreach ($zone in (Get-DnsServerZone | Where-Object {-not $_.IsAutoCreated})) { Write-AzsReadinessLog -Message "Removing DNS zone '$($zone.ZoneName)'." Remove-DnsServerZone -Name $zone.ZoneName -Force } } catch { Write-AzsReadinessLog -Message "Failed removing DNS server zone. Error: $($_.Exception.Message)" -Type 'Error' } } if ( ($dnsService = Get-Service | Where-Object {$_.Name -eq 'Dns'}) -and ($dnsService.Status -ne [System.ServiceProcess.ServiceControllerStatus]::Stopped) ) { Write-AzsReadinessLog -Message 'Stopping DNS server service.' Stop-Service -Name 'Dns' -WarningAction SilentlyContinue } if ($Uninstall) { $windowsFeatures = Get-WindowsFeature | Where-Object {$_.Installed} if ($windowsFeatures.Name -contains 'RSAT-DNS-Server') { Write-AzsReadinessLog -Message "Uninstalling DNS management tools." $null = Uninstall-WindowsFeature -Name 'RSAT-DNS-Server' -WarningAction SilentlyContinue } if ($windowsFeatures.Name -contains 'DNS') { Write-AzsReadinessLog -Message "Uninstalling DNS server." $null = Uninstall-WindowsFeature -Name 'DNS' -WarningAction SilentlyContinue } } } catch { $errorMessage = "Failed removing DNS server. Error: $($_.Exception.Message)" Write-AzsReadinessLog -Message $errorMessage -Type 'Error' throw $errorMessage } } <# .SYNOPSIS Applies configuration to the virtual router. .DESCRIPTION Used in virtual router scenarios. #> function Set-VirtualRouterConfig { [CmdletBinding()] param ( [Parameter()] [System.String] $ComputerName, [Parameter(Mandatory = $true)] [System.Management.Automation.PSCredential] $Credential, [Parameter()] [SSH.SshSession] $SSHSession, [Parameter()] [switch] $Clear, [Parameter()] [System.Int32] $LocalAsn, [Parameter()] [System.Net.IPAddress] $LoopbackAddress, [Parameter()] [System.Object[]] $InterfaceConfiguration, [Parameter()] [System.Object[]] $BgpNeighborConfiguration, [Parameter()] [System.String] $BmcNetworkInterfaceName = 'Ethernet0', [Parameter()] [System.Net.IPAddress] $BmcNetworkAddress, [Parameter()] [System.Int32] $BmcNetworkPrefixLength, [Parameter()] [System.Int32] $BmcNetworkVlanId = 125, [Parameter()] [System.String] $ExternalNetworkInterfaceName = 'Ethernet4', [Parameter()] [System.Net.IPAddress] $ExternalNetworkAddress, [Parameter()] [System.Int32] $ExternalNetworkPrefixLength, [Parameter()] [System.Int32] $ExternalNetworkVlanId = 1000, [Parameter()] [System.String] $TestDataPath ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop try { $workingDirectory = Join-Path -Path $env:TEMP -ChildPath (New-Guid) Write-AzsReadinessLog -Message "Using temporary folder $workingDirectory" $null = New-Item -Path $workingDirectory -Itemtype Directory if (-not $ComputerName) { $ComputerName = $SSHSession.Host } if ($SSHSession.Host -eq $Computername -and $SSHSession.Connected) { Write-AzsReadinessLog -Message "Using existing SSH session" $ComputerName = $SSHSession.Host } else { Write-AzsReadinessLog -Message "Establishing new SSH session with $ComputerName" $SSHSession = Assert-VirtualRouterSession -ComputerName $ComputerName -Credential $Credential } Write-AzsReadinessLog -Message "Retrieving current configuration file" $localFilePath = Join-Path -Path $workingDirectory -ChildPath 'config_db.json' Get-SCPItem -ComputerName $ComputerName -Credential $Credential -AcceptKey:$true -Path '/etc/sonic/config_db.json' -PathType File -Destination $workingDirectory $config = Get-Content -Path $localFilePath -Raw | ConvertFrom-Json if ($Clear) { Write-AzsReadinessLog -Message "Clearing configuration" $newConfig = [PSCustomObject]@{ DEVICE_METADATA = $config.DEVICE_METADATA PORT = $config.PORT LOOPBACK_INTERFACE = @{} INTERFACE = @{} BGP_NEIGHBOR = @{} VLAN = @{} VLAN_MEMBER = @{} VLAN_INTERFACE= @{} } } else { $newConfig = $config } # Generate a new random MAC address for the device $arrMac = $newConfig.DEVICE_METADATA.localhost.mac -split ':' for ($i = 3; $i -le 5; $i++) { $arrMac[$i] = [System.String]::Format('{0:X2}', (Get-Random -Maximum 255)) } $newConfig.DEVICE_METADATA.localhost.mac = $arrMac -join ':' if ($LocalAsn) { Write-AzsReadinessLog -Message "Local ASN = $LocalAsn" $newConfig.DEVICE_METADATA.localhost.bgp_asn = $LocalAsn } if ($LoopbackAddress) { Write-AzsReadinessLog -Message "Loopback address = $LoopbackAddress" $newConfig.LOOPBACK_INTERFACE = [PSCustomObject]@{ "Loopback0|$LoopbackAddress/32" = @{} } } $interfaceHash = @{} $bgp_neighborHash = @{} $staticPeers = @() foreach ($interface in $InterfaceConfiguration) { Write-AzsReadinessLog -Message "Interface $($interface.Name) address = $($interface.IPAddress)/$($interface.PrefixLength)" $interfaceHash."$($interface.Name)|$($interface.IPAddress)/$($interface.PrefixLength)" = @{} } foreach ($bgpNeighbor in $BgpNeighborConfiguration) { if ($bgpNeighbor.RemoteAsn) { Write-AzsReadinessLog -Message "BGP Neighbor address = $($bgpNeighbor.RemoteAddress) with ASN = $($bgpNeighbor.RemoteAsn)" $bgp_neighborHash."$($bgpNeighbor.RemoteAddress)" = @{ local_addr = $bgpNeighbor.LocalAddress asn = $bgpNeighbor.RemoteAsn name = $bgpNeighbor.Name } } else { $staticPeers += $bgpNeighbor.RemoteAddress } } $newConfig.INTERFACE = [PSCustomObject]$interfaceHash $newConfig.BGP_NEIGHBOR = [PSCustomObject]$bgp_neighborHash if ($BmcNetworkAddress -and $ExternalNetworkAddress) { $newConfig.VLAN = [PSCustomObject]@{ "Vlan$BmcNetworkVlanId" = @{ members = @($BmcNetworkInterfaceName) vlanid = $BmcNetworkVlanId } "Vlan$ExternalNetworkVlanId" = @{ members = @($ExternalNetworkInterfaceName) vlanid = $ExternalNetworkVlanId } } $newConfig.VLAN_MEMBER = [PSCustomObject]@{ "Vlan$BmcNetworkVlanId|$BmcNetworkInterfaceName" = @{ tagging_mode = "untagged" } "Vlan$ExternalNetworkVlanId|$ExternalNetworkInterfaceName" = @{ tagging_mode = "untagged" } } $newConfig.VLAN_INTERFACE = [PSCustomObject]@{ "Vlan$BmcNetworkVlanId|$BmcNetworkAddress/$BmcNetworkPrefixLength" = @{} "Vlan$ExternalNetworkVlanId|$ExternalNetworkAddress/$ExternalNetworkPrefixLength" = @{} } } Write-AzsReadinessLog -Message "Saving updated configuration file" Set-Content -Value ($newConfig | ConvertTo-Json).Replace("`r`n","`n") -Path $localFilePath -NoNewline Set-SCPItem -ComputerName $ComputerName -Credential $Credential -AcceptKey:$true -Path $localFilePath -Destination '/home/admin' $sshOutput = Invoke-VirtualRouterCommand -Command 'sudo cp /home/admin/config_db.json /etc/sonic' -SSHSession $SSHSession -Credential $Credential $sshOutput = Invoke-VirtualRouterCommand -Command 'sudo config reload -y -f' -SSHSession $SSHSession -Credential $Credential -Timeout 120 foreach ($line in $sshOutput.Output) { Write-AzsReadinessLog -Message $line } # Copy the python script and <cloudid>-TestData.json to the SonicVM if ($TestDataPath) { $pythonFile = Join-Path -Path $PSScriptRoot -ChildPath 'Microsoft.AzureStack.ReadinessChecker.NetworkValidation.py' Write-AzsReadinessLog -Message "Copying file $pythonFile to the Sonic VM." Set-SCPItem -ComputerName $ComputerName -Credential $Credential -AcceptKey:$true -Path $pythonFile -Destination '/home/admin' foreach ($testData in (Get-ChildItem -Path $TestDataPath -Filter "*-TestData.json")) { Write-AzsReadinessLog -Message "Copying file $($testData.FullName) to the Sonic VM." Set-SCPItem -ComputerName $ComputerName -Credential $Credential -AcceptKey:$true -Path $testData.FullName -Destination '/home/admin' } } if ($staticPeers) { Write-AzsReadinessLog -Message "Configuring static routes" foreach ($peerAddress in $staticPeers) { $sshOutput = Invoke-VirtualRouterCommand -Command "sudo config route add prefix 0.0.0.0/0 nexthop $peerAddress" -SSHSession $SSHSession -Credential $Credential } } } catch { $file = $_.InvocationInfo.ScriptName $line = $_.InvocationInfo.ScriptLineNumber $exceptionMessage = $_.Exception.Message $message = "$file : $line >> $exceptionMessage" Write-AzsReadinessLog -Message "Error occured in Set-VirtualRouterConfig: $message" -Type 'Error' -ToScreen throw $_ } finally { if (Test-Path -Path $workingDirectory) { Write-AzsReadinessLog -Message "Removing temporary folder $workingDirectory" Remove-Item -Path $workingDirectory -Recurse -Force } } } <# .SYNOPSIS Runs a command on the virtual router and returns formatted output. .DESCRIPTION Used in virtual router scenarios. #> function Invoke-VirtualRouterCommand { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [SSH.SshSession] $SSHSession, [Parameter(Mandatory = $true)] [System.Management.Automation.PSCredential] $Credential, [Parameter(Mandatory = $true)] [System.String] $Command, [Parameter()] [System.Int32] $Timeout = 10, [Parameter()] [System.Int32[]] $SuccessCodes = 0 ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop try { if ($SSHSession.Connected) { Write-AzsReadinessLog -Message "Using existing SSH session" } else { $computerName = $SSHSession.Host Write-AzsReadinessLog -Message "Establishing new SSH session with $computerName" $SSHSession = Assert-VirtualRouterSession -ComputerName $computerName -Credential $Credential } $retry = 0 do { Write-AzsReadinessLog -Message "Invoking command '$Command' on $($SSHSession.Host)" $sshOutput = Invoke-SSHCommand -Command $Command -SSHSession $SSHSession -Timeout $Timeout -EnsureConnection Write-AzsReadinessLog -Message "Command returned status code $($sshOutput.ExitStatus)" if ($sshOutput.ExitStatus -notin $SuccessCodes) { $retry++ Write-AzsReadinessLog -Message "Sleeping for 5 seconds" Start-Sleep -Seconds 5 } } until (($sshOutput.ExitStatus -in $SuccessCodes) -or ($retry -ge 10)) foreach ($line in $sshOutput.Output) { Write-AzsReadinessLog -Message $line } if ($sshOutput.ExitStatus -notin $SuccessCodes) { throw "Error running SSH command, status code = $($sshOutput.ExitStatus)" } return $sshOutput.Output } catch { Write-AzsReadinessLog -Message "Error occured: $($_.Exception.Message)" -Type 'Error' throw $_ } } <# .SYNOPSIS Establishes SSH session with the virtual router with retries. .DESCRIPTION Used in virtual router scenarios. #> function Assert-VirtualRouterSession { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $ComputerName, [Parameter(Mandatory = $true)] [System.Management.Automation.PSCredential] $Credential, [Parameter()] [System.Int32] $NumberOfRetries = 30 ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $retry = 0 $ssh = $false do { Write-AzsReadinessLog -Message "Attempting to establish a new SSH session with $ComputerName" try { $ssh = New-SSHSession -ComputerName $ComputerName -Credential $Credential -AcceptKey:$true Write-AzsReadinessLog -Message "SSH session established" } catch { $retry++ $exception = $_ Write-AzsReadinessLog -Message "Failed to establish SSH session with $ComputerName. Retry = $retry" Write-AzsReadinessLog -Message "Posh-SSH Exception: $($exception.Exception.Message)" Start-Sleep -Seconds 5 } } until ($ssh -or $retry -ge $NumberOfRetries) if (-not $ssh) { Write-AzsReadinessLog -Message "Unable to establish SSH session with $ComputerName after $retry retries" -Type 'Error' throw $($exception.Exception.Message) } return $ssh } <# .SYNOPSIS Configures the external connection on the virtual router. .DESCRIPTION Used in virtual router scenarios. #> function Connect-VirtualRouterUplink { [CmdletBinding()] param ( [Parameter()] [System.String] $VMName = 'TOR', [Parameter(Mandatory = $true)] [System.String] $NetAdapterInterfaceDescription, [Parameter()] [System.String] $VNicName = 'Ethernet8' ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop try { if (-not ($netAdapter = Get-NetAdapter | Where-Object {$_.InterfaceDescription -eq $NetAdapterInterfaceDescription})) { throw "Network adapter with description $NetAdapterInterfaceDescription not found" } if ($vmSwitch = Get-VMSwitch | Where-Object {$_.NetAdapterInterfaceDescription -eq $NetAdapterInterfaceDescription}) { Write-AzsReadinessLog -Message "Using existing Hyper-V switch $($vmSwitch.Name)" } else { $vmSwitchName = "TOR-Uplink-$($netAdapter.InterfaceIndex)" $retry = 1 do { try { Write-AzsReadinessLog -Message "Creating new Hyper-V external switch $vmSwitchName, attempt $retry" $vmSwitch = New-VMSwitch -Name $vmSwitchName -NetAdapterInterfaceDescription $NetAdapterInterfaceDescription -AllowManagementOS $false $retry = 10 } catch { Start-Sleep -Seconds 1 $retry++ if ($retry -eq 10) { throw $_ } } } until ($retry -eq 10) } Write-AzsReadinessLog -Message "Connecting virtual machine $VMName interface $VNicName to the external Hyper-V switch $($vmSwitch.Name)" Connect-VMNetworkAdapter -VMName $VMName -Name $VNicName -SwitchName $vmSwitch.Name } catch { Write-AzsReadinessLog -Message "Error occured: $($_.Exception.Message)" -Type 'Error' throw $_ } } <# .SYNOPSIS Sends ICMP pings with variable buffer size to discover network path MTU. .DESCRIPTION Many network devices block ICMP messages for perceived security benefits, including the errors that are necessary for the proper operation of PMTUD. As such, this function relies only on successful echo responses by the destination host, and treats any ICMP error, including timeout, as Fragmentation Required. #> function Invoke-PathMtuDiscovery { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [System.String] $Destination, [Parameter()] [ValidateRange(32, 12000)] [System.Int32] $MinimumBuffer = 32, [Parameter()] [ValidateRange(32, 12000)] [System.Int32] $Mtu = 1500 ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $overhead = 28 # Number of bytes reserved for the IP(20) and ICMP(8) packet headers $low = $MinimumBuffer $high = $Mtu - $overhead try { Write-AzsReadinessLog -Message "Discovering path MTU to $Destination" $ping = New-Object -TypeName System.Net.NetworkInformation.Ping $options = New-Object -TypeName System.Net.NetworkInformation.PingOptions -ArgumentList 64, $true $timeoutMs = 5000 $buffer = 1..$low | ForEach-Object -Process {[byte]65} $lowerTry = $ping.Send($Destination, $timeoutMs, $buffer, $options) if ($lowerTry.Status -ne [System.Net.NetworkInformation.IPStatus]::Success) { Write-AzsReadinessLog -Message "Destination $Destination is unreachable" -Type 'Error' return 0 } $buffer = 1..$high | ForEach-Object -Process {[byte]65} $upperTry = $ping.Send($Destination, $timeoutMs, $buffer, $options) if ($upperTry.Status -eq [System.Net.NetworkInformation.IPStatus]::Success) { Write-AzsReadinessLog -Message "Ping to $Destination succeeded with the maximum buffer size of $high" return $Mtu } $timeoutMs = 1000 Write-AzsReadinessLog -Message "Discovering path MTU to $Destination using ICMP buffer size between $low and $high bytes" do { $size = [System.Math]::Truncate(($low + $high)/2) $buffer = 1..$size | ForEach-Object -Process {[byte]65} $try = $ping.Send($Destination, $timeoutMs, $buffer, $options) Write-AzsReadinessLog -Message "Ping to $Destination with buffer size $size status $($try.Status)" if ($try.Status -eq [System.Net.NetworkInformation.IPStatus]::Success) { $low = $size } else { $high = $size } } until ($high - $low -eq 1 -and $try.Status -eq [System.Net.NetworkInformation.IPStatus]::Success) Write-AzsReadinessLog -Message "Discovered maximum ICMP buffer size is $size. Network path MTU is $($size + $overhead) bytes." return $size + $overhead } catch { Write-AzsReadinessLog -Message "Error occured: $($_.Exception.Message)" -Type 'Error' throw $_ } } <# .SYNOPSIS Creates the <CloudID>-TestData.json that is used by the Python Test Script that runs on the SonicVM .DESCRIPTION The test data is a hash table that contains all the data, and tests that need to be prefromed for validating the Public VIPS network. #> function Set-TestDataForSonicVM { [CmdletBinding()] param ( # Full Json [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] [System.Object] $DeploymentData, # Azure Resource Manager endpoint URI for custom cloud [Parameter()] [System.Uri] $CustomCloudArmEndpoint, # Path for saving the test data file [Parameter(Mandatory = $true)] [System.String] $OutputPath ) try { $cloudID = Get-CloudID -DeploymentData $DeploymentData $cloudInfo = $DeploymentData.ConfigData.InputData.Cloud | Where-Object {$_.Id -eq $cloudID} $ntpServers = $cloudInfo.TimeServer $dnsServers = $cloudInfo.DNSForwarder $connectToAzure = $cloudInfo.ConnectToAzure $azureEnv = $cloudInfo.InfraAzureEnvironment $extnernalVipNet = $DeploymentData.ConfigData.IPConfigData | Where-Object {$_.Name -match "$($cloudID)-External-VIPS"} $sourceIP = (Get-IPAddressOffset -SubnetAddress $extnernalVipNet.IPv4NetworkAddress -Offset 15).IPAddressToString $bmcNet = $DeploymentData.ConfigData.IPConfigData | Where-Object {$_.Name -match "Rack01-BMCMgmt"} $gateway = $bmcNet.IPv4Gateway $loopbacks = $DeploymentData.ConfigData.IPConfigData | Where-Object {$_.Name -match "Loopback"} if ($connectToAzure -eq "Azure Active Directory") { if ($CustomCloudArmEndpoint) { $endpoints = @{ARMUri = $CustomCloudArmEndpoint} } else { $endpoints = Get-AzureURIs -AzureEnvironment $azureEnv } } $count = 1 $testData = [ordered]@{ "cloudId" = $cloudID; "gateway" = $gateway; "neighbors" = [ordered]@{} "sourceIP" = @{ "ipaddress" = $sourceIP } "tests" = [ordered]@{ [string]$count++ = [ordered]@{ "testName" = "ntpServer" "destination" = $ntpServers "endPoints" = $false } } } # Only for AAD perform the following tests if ($endpoints) { $testData.tests += [ordered]@{ [string]$count++ = [ordered]@{ "testName" = "dnsServer" "destination" = $dnsServers "endPoints" = [System.Collections.ArrayList]$endpoints.Values } [string]$count++ = [ordered]@{ "testName" = "webRequests" "destination" = $false "endPoints" = [System.Collections.ArrayList]$endpoints.Values } } } foreach ($neighbor in $loopbacks) { $name = ($neighbor.Name -replace ".*/").ToLower() $testData.neighbors += @{ $name = @{ "ipaddress" = $neighbor.IPv4FirstAddress } } } $tdJson = $testData | ConvertTo-Json -Depth 100 $utf8NoBomEncoding = New-Object -TypeName System.Text.UTF8Encoding -ArgumentList $false $outputFilePath = Join-Path -Path $OutputPath -ChildPath "$cloudID-TestData.json" [System.IO.File]::WriteAllLines($outputFilePath, $tdJson, $utf8NoBomEncoding) return $testData } catch { $file = $_.InvocationInfo.ScriptName $line = $_.InvocationInfo.ScriptLineNumber $message = $_.Exception.Message Write-AzsReadinessLog -Message "ERROR: $file : $line" -Type 'Error' Write-AzsReadinessLog -Message $message -Type 'Error' } } <# .SYNOPSIS Obtains the Cloud ID from the DeploymentData json based on the DVM .DESCRIPTION The DVM has a cloud ID that is unique to it. Knowing the Cloud ID is needed to support "Lab Scenarios" #> function Get-CloudID { [CmdletBinding()] param ( # Full Json [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] [System.Object] $DeploymentData ) $dvmIp = $DeploymentData.ScaleUnits.DeploymentData.DeploymentVM.IPAddress $assignments = ($DeploymentData.ConfigData.IPConfigData | Where-Object { $_.Name -eq "Rack01-BMCMgmt" }).Assignments $cloudId = ($assignments | Where-Object {$_.IPv4Address -eq $dvmIp}).CloudId if (-not $cloudId){ Write-AzsReadinessLog -Message "No cloud id found." -Type 'Error' } return $cloudId } function Invoke-ScopeCleanup { [CmdletBinding()] param ( # Full Deployment Data Json [Parameter(Mandatory = $True)] [ValidateNotNullOrEmpty()] [System.Object] $DeploymentData ) $cloudID = Get-CloudID -DeploymentData $DeploymentData $mgmtNetworkName = "$cloudID-NRC-Mgmt" $mgmtNetwork = $DeploymentData.ConfigData.IPConfigData | Where-Object {$_.Name -match $mgmtNetworkName} try { $dhcpIpAddresses = $null Write-AzsReadinessLog -Message "Getting DHCP assigned IP addresses" $dhcpIpAddresses = (Get-DhcpServerv4Lease -ScopeId $mgmtNetwork.IPv4NetworkAddress).IPAddress.IPAddressToString } catch { Write-AzsReadinessLog -Message "Failed getting DHCP assigned addresses. Error: $($_.Exception.Message)" } foreach ($dhcpIpAddress in $dhcpIpAddresses) { Write-AzsReadinessLog -Message "Testing DHCP-assigned IP address $dhcpIpAddress" if (Test-Connection -ComputerName $dhcpIpAddress -Count 1 -Quiet) { $vmMgmtIP = $dhcpIpAddress Write-AzsReadinessLog -Message "Ping to the virtual router at $vmMgmtIP successful" } else { $null = Remove-DhcpServerv4Lease -IPAddress $dhcpIpAddress -ErrorAction SilentlyContinue } } } <# .SYNOPSIS Convert subnet mask to prefix length .DESCRIPTION Converts the subnet mask string value (i.e. 255.255.255.192) to integer subnet prefix length, i.e 26 #> function Convert-SubnetMaskToPrefixLength { [CmdletBinding()] param ( [System.Net.IPAddress] $SubnetMask ) $allowedValues = @(0, 128, 192, 224, 240, 248, 252, 254, 255) $addressBytes = $SubnetMask.GetAddressBytes() $lastByte = 255 foreach ($byte in $addressBytes) { if ($byte -notin $allowedValues -or ($lastByte -ne 255 -and $byte -gt 0)) { throw "Invalid SubnetMask specified [$SubnetMask]" } [System.String]$binaryString += [System.Convert]::ToString($byte, 2) $lastByte = $byte } return $binaryString.TrimEnd('0').Length } <# .SYNOPSIS Test for PowerShell FIPS mode. .DESCRIPTION Used for temporarily disabling FIPS mode. #> function Test-FIPSMode { [CmdletBinding()] [OutputType([System.Boolean])] param () $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Write-AzsReadinessLog -Message 'Testing for PowerShell FIPS mode.' try { $null = New-Object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider Write-AzsReadinessLog -Message 'PowerShell FIPS mode is disabled.' $result = $false } catch { Write-AzsReadinessLog -Message 'PowerShell FIPS mode is enabled.' $result = $true } return $result } <# .SYNOPSIS Sets FIPS mode and returns FIPS mode setting prior to change. .DESCRIPTION Used for temporarily disabling FIPS mode. .PARAMETER Enabled Whether or not FIPS should be enabled. #> function Reset-FIPSMode { [CmdletBinding()] [OutputType([System.Boolean])] param ( [Parameter(Mandatory = $true)] [System.Boolean] $Enabled ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop # Return the FIPS mode state before we reset $result = (Get-ItemProperty -Path 'HKLM:SYSTEM\CurrentControlSet\Control\Lsa\FipsAlgorithmPolicy').Enabled -eq 1 Write-AzsReadinessLog -Message "Current FipsAlgorithmPolicy setting is '$([int]$result)'." # Set the FIPS mode state if necessary if ($Enabled -ne $result) { Write-AzsReadinessLog -Message "Setting FipsAlgorithmPolicy to '$([int]$enabled)'." Set-ItemProperty -Path 'HKLM:SYSTEM\CurrentControlSet\Control\Lsa\FipsAlgorithmPolicy' -Name Enabled -Value ([int]$enabled) } return $result } <# .SYNOPSIS Determines if a cmdlets needs to be restarted after changing some element of the PowerShell environment. .DESCRIPTION Tests specific PowerShell environment settings, and returns encoded command to be used for restart. .PARAMETER Parameters Parameters from the original cmdlet. .PARAMETER ModuleName Name of the module with the original cmdlet to be restarted. .PARAMETER FIPS Test for PowerShell FIPS mode. #> function Invoke-CmdletRestart { [CmdletBinding()] [OutputType([System.Collections.Hashtable])] param ( [Parameter(Mandatory = $false)] [System.Collections.Hashtable] $Parameters, [Parameter(Mandatory = $false)] [System.String] $ModuleName = $script:moduleName, [Switch] $FIPS ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $functionName = (Get-PSCallStack)[1].Command $cmdletRestart = @{ Restart = $false } $command = '' # Test whether a restart is required Write-AzsReadinessLog -Message "Testing whether a cmdlet '$functionName' restart is required." if ($FIPS.IsPresent) { if (Test-FIPSMode) { $cmdletRestart.Restart = $true $cmdletRestart.FIPSMode = Reset-FIPSMode -Enabled $false } } # If a restart is required, build the command line to invoke if ($cmdletRestart.Restart) { Write-AzsReadinessLog -Message "Cmdlet '$functionName' restart is required." # Import the module $command += "Import-Module $PSScriptRoot\..\$ModuleName.psd1; " # Execute the cmdlet $command += "$functionName" # Build parameters if ($PSBoundParameters.ContainsKey('Parameters')) { foreach ($key in $Parameters.Keys) { $command += " -$key " switch ($Parameters.$key.GetType()) { 'PSCredential' { $command += "`$(New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList '$($Parameters.$key.UserName)',(ConvertTo-SecureString -String '$($Parameters.$key.GetNetworkCredential().Password)' -AsPlainText -Force))" } 'String[]' { $command += "@('$($Parameters.$key -join "', '")')" } 'Uri[]' { $command += "@('$($Parameters.$key.OriginalString -join "', '")')" } default { $command += $Parameters.$key } } } } # Return the command in encoded format $encodedCommand = [Convert]::ToBase64String([System.Text.Encoding]::Unicode.GetBytes($command)) $cmdletRestart.EncodedCommand = $encodedCommand } else { Write-AzsReadinessLog -Message "Cmdlet '$functionName' restart is not required." } return $cmdletRestart } # SIG # Begin signature block # MIIjhQYJKoZIhvcNAQcCoIIjdjCCI3ICAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDcj/tERGY039sI # 3eLI6lLJUlcS5N/vItRKmi/RYAznDaCCDYEwggX/MIID56ADAgECAhMzAAACUosz # qviV8znbAAAAAAJSMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjEwOTAyMTgzMjU5WhcNMjIwOTAxMTgzMjU5WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDQ5M+Ps/X7BNuv5B/0I6uoDwj0NJOo1KrVQqO7ggRXccklyTrWL4xMShjIou2I # sbYnF67wXzVAq5Om4oe+LfzSDOzjcb6ms00gBo0OQaqwQ1BijyJ7NvDf80I1fW9O # L76Kt0Wpc2zrGhzcHdb7upPrvxvSNNUvxK3sgw7YTt31410vpEp8yfBEl/hd8ZzA # v47DCgJ5j1zm295s1RVZHNp6MoiQFVOECm4AwK2l28i+YER1JO4IplTH44uvzX9o # RnJHaMvWzZEpozPy4jNO2DDqbcNs4zh7AWMhE1PWFVA+CHI/En5nASvCvLmuR/t8 # q4bc8XR8QIZJQSp+2U6m2ldNAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUNZJaEUGL2Guwt7ZOAu4efEYXedEw # UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1 # ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDY3NTk3MB8GA1UdIwQYMBaAFEhu # ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu # bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w # Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx # MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAFkk3 # uSxkTEBh1NtAl7BivIEsAWdgX1qZ+EdZMYbQKasY6IhSLXRMxF1B3OKdR9K/kccp # kvNcGl8D7YyYS4mhCUMBR+VLrg3f8PUj38A9V5aiY2/Jok7WZFOAmjPRNNGnyeg7 # l0lTiThFqE+2aOs6+heegqAdelGgNJKRHLWRuhGKuLIw5lkgx9Ky+QvZrn/Ddi8u # TIgWKp+MGG8xY6PBvvjgt9jQShlnPrZ3UY8Bvwy6rynhXBaV0V0TTL0gEx7eh/K1 # o8Miaru6s/7FyqOLeUS4vTHh9TgBL5DtxCYurXbSBVtL1Fj44+Od/6cmC9mmvrti # yG709Y3Rd3YdJj2f3GJq7Y7KdWq0QYhatKhBeg4fxjhg0yut2g6aM1mxjNPrE48z # 6HWCNGu9gMK5ZudldRw4a45Z06Aoktof0CqOyTErvq0YjoE4Xpa0+87T/PVUXNqf # 7Y+qSU7+9LtLQuMYR4w3cSPjuNusvLf9gBnch5RqM7kaDtYWDgLyB42EfsxeMqwK # WwA+TVi0HrWRqfSx2olbE56hJcEkMjOSKz3sRuupFCX3UroyYf52L+2iVTrda8XW # esPG62Mnn3T8AuLfzeJFuAbfOSERx7IFZO92UPoXE1uEjL5skl1yTZB3MubgOA4F # 8KoRNhviFAEST+nG8c8uIsbZeb08SeYQMqjVEmkwggd6MIIFYqADAgECAgphDpDS # AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 # ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla # MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT # H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG # OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S # 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz # y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7 # 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u # M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33 # X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl # XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP # 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB # l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF # RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM # CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ # BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud # DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO # 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0 # LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB # FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw # cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA # XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY # 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj # 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd # d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ # Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf # wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ # aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j # NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B # xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96 # eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7 # r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I # RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIVWjCCFVYCAQEwgZUwfjELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z # b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAlKLM6r4lfM52wAAAAACUjAN # BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor # BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQgc8/TxfGJ # Zou03NU/zDtHwwHktNUFlbPlOgSN2rURxbcwQgYKKwYBBAGCNwIBDDE0MDKgFIAS # AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN # BgkqhkiG9w0BAQEFAASCAQBXrC8s4s2MM4i8b7cyk0Mw7/lHyzGqaGJkA61zTJuk # YDiasSMJdU0p3OYzGV7h43byZ+xoa2kSnM9+7Y2GFo13uFBdA2Xb617Zc0m8VTbQ # c+rj1mqfFkk8r4mUBAj75RKAlU++X96GNTT3kac/TiM97Z9aUbl6dQckHbeQ09X/ # j7kQVnFw9e/OAMVR3uChHkMF96UAiPe0gsrulRtb5iJaPDBWcGFPUqdGXEAGdcAk # gXQ7yzm0MhcxBDFLrLZdKHZwYugNnZuFGt7ruNt4Sq1iKZZxqf2j3QSbaqkgiWpI # 2ADnQWBo9NSibyUT48NJDoH1ZiMJBGJhgLx6XU1dhVjmoYIS5DCCEuAGCisGAQQB # gjcDAwExghLQMIISzAYJKoZIhvcNAQcCoIISvTCCErkCAQMxDzANBglghkgBZQME # AgEFADCCAVEGCyqGSIb3DQEJEAEEoIIBQASCATwwggE4AgEBBgorBgEEAYRZCgMB # MDEwDQYJYIZIAWUDBAIBBQAEIHDRyWJjg+BuWoaMblhg07qPrRQAM7auOUGhF5fu # t8FZAgZhktXwXB8YEzIwMjExMTE2MDg0OTQ0Ljk3NVowBIACAfSggdCkgc0wgcox # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJTAjBgNVBAsTHE1p # Y3Jvc29mdCBBbWVyaWNhIE9wZXJhdGlvbnMxJjAkBgNVBAsTHVRoYWxlcyBUU1Mg # RVNOOkU1QTYtRTI3Qy01OTJFMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFt # cCBTZXJ2aWNloIIOOzCCBPEwggPZoAMCAQICEzMAAAFHnY/x5t4xg1kAAAAAAUcw # DQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0 # b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3Jh # dGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcN # MjAxMTEyMTgyNTU1WhcNMjIwMjExMTgyNTU1WjCByjELMAkGA1UEBhMCVVMxEzAR # BgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1p # Y3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2Eg # T3BlcmF0aW9uczEmMCQGA1UECxMdVGhhbGVzIFRTUyBFU046RTVBNi1FMjdDLTU5 # MkUxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggEiMA0G # CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtBQNM6X32KFk/BJ8YaprfzEt6Lj34 # G+VLjzgfEgOGSVd1Mu7nCphK0K4oyPrzItgNRjB4gUiKq6GzgxdDHgZPgTEvm57z # sascyGrybWkf3VVr8bqf2PIgGvwKDNEgVcygsEbuWwXz9Li6M7AOoD4TB8fl4ATm # +L7b4+lYDUMJYMLzpiJzM745a0XHiriUaOpYWfkwO9Hz6uf+k2Hq7yGyguH8naPL # MnYfmYIt2PXAwWVvG4MD4YbjXBVZ14ueh7YlqZTMua3n9kT1CZDsHvz+o58nsoam # XRwRFOb7LDjVV++cZIZLO29usiI0H79tb3fSvh9tU7QC7CirNCBYagNJAgMBAAGj # ggEbMIIBFzAdBgNVHQ4EFgQUtPjcb95koYZXGy9DPxN49dSCsLowHwYDVR0jBBgw # FoAU1WM6XIoxkPNDe3xGG8UzaFqFbVUwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDov # L2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljVGltU3RhUENB # XzIwMTAtMDctMDEuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0 # cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNUaW1TdGFQQ0FfMjAx # MC0wNy0wMS5jcnQwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggrBgEFBQcDCDAN # BgkqhkiG9w0BAQsFAAOCAQEAUMQOyjV+ea2kEtXqD0cOfD2Z2PFUIy5kLkGU53RD # GcfhlzIR9QlTgZLqTEhgLLuCSy6jcma+nPg7e5Xg1oqCZcZJRwtRPzS1F6/M6YR3 # 5H3brN0maVnPrmrQ91kkfsNqDTtuWDiAIBfkNEgCpQZCb4OV3HMu5L8eZzg5dUaJ # 7XE+LBuphJSLFJtabxYt4fkCQxnTD2z50Y32ZuXiNmFFia7qVq+3Yc3mmW02+/KW # H8P1HPiobJG8crGYgSEkxtkUXGdoutwGWW88KR9RRcM/4GKLqt2OQ8AWEQb7shgM # 8pxNvu30TxejRApa4WAfOAejTG4+KzBm67XjVZ2IlXAPkjCCBnEwggRZoAMCAQIC # CmEJgSoAAAAAAAIwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRp # ZmljYXRlIEF1dGhvcml0eSAyMDEwMB4XDTEwMDcwMTIxMzY1NVoXDTI1MDcwMTIx # NDY1NVowfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNV # BAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQG # A1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwggEiMA0GCSqGSIb3 # DQEBAQUAA4IBDwAwggEKAoIBAQCpHQ28dxGKOiDs/BOX9fp/aZRrdFQQ1aUKAIKF # ++18aEssX8XD5WHCdrc+Zitb8BVTJwQxH0EbGpUdzgkTjnxhMFmxMEQP8WCIhFRD # DNdNuDgIs0Ldk6zWczBXJoKjRQ3Q6vVHgc2/JGAyWGBG8lhHhjKEHnRhZ5FfgVSx # z5NMksHEpl3RYRNuKMYa+YaAu99h/EbBJx0kZxJyGiGKr0tkiVBisV39dx898Fd1 # rL2KQk1AUdEPnAY+Z3/1ZsADlkR+79BL/W7lmsqxqPJ6Kgox8NpOBpG2iAg16Hgc # sOmZzTznL0S6p/TcZL2kAcEgCZN4zfy8wMlEXV4WnAEFTyJNAgMBAAGjggHmMIIB # 4jAQBgkrBgEEAYI3FQEEAwIBADAdBgNVHQ4EFgQU1WM6XIoxkPNDe3xGG8UzaFqF # bVUwGQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwCwYDVR0PBAQDAgGGMA8GA1Ud # EwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU1fZWy4/oolxiaNE9lJBb186aGMQwVgYD # VR0fBE8wTTBLoEmgR4ZFaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwv # cHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3JsMFoGCCsGAQUFBwEB # BE4wTDBKBggrBgEFBQcwAoY+aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9j # ZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcnQwgaAGA1UdIAEB/wSBlTCB # kjCBjwYJKwYBBAGCNy4DMIGBMD0GCCsGAQUFBwIBFjFodHRwOi8vd3d3Lm1pY3Jv # c29mdC5jb20vUEtJL2RvY3MvQ1BTL2RlZmF1bHQuaHRtMEAGCCsGAQUFBwICMDQe # MiAdAEwAZQBnAGEAbABfAFAAbwBsAGkAYwB5AF8AUwB0AGEAdABlAG0AZQBuAHQA # LiAdMA0GCSqGSIb3DQEBCwUAA4ICAQAH5ohRDeLG4Jg/gXEDPZ2joSFvs+umzPUx # vs8F4qn++ldtGTCzwsVmyWrf9efweL3HqJ4l4/m87WtUVwgrUYJEEvu5U4zM9GAS # inbMQEBBm9xcF/9c+V4XNZgkVkt070IQyK+/f8Z/8jd9Wj8c8pl5SpFSAK84Dxf1 # L3mBZdmptWvkx872ynoAb0swRCQiPM/tA6WWj1kpvLb9BOFwnzJKJ/1Vry/+tuWO # M7tiX5rbV0Dp8c6ZZpCM/2pif93FSguRJuI57BlKcWOdeyFtw5yjojz6f32WapB4 # pm3S4Zz5Hfw42JT0xqUKloakvZ4argRCg7i1gJsiOCC1JeVk7Pf0v35jWSUPei45 # V3aicaoGig+JFrphpxHLmtgOR5qAxdDNp9DvfYPw4TtxCd9ddJgiCGHasFAeb73x # 4QDf5zEHpJM692VHeOj4qEir995yfmFrb3epgcunCaw5u+zGy9iCtHLNHfS4hQEe # gPsbiSpUObJb2sgNVZl6h3M7COaYLeqN4DMuEin1wC9UJyH3yKxO2ii4sanblrKn # QqLJzxlBTeCG+SqaoxFmMNO7dDJL32N79ZmKLxvHIa9Zta7cRDyXUHHXodLFVeNp # 3lfB0d4wwP3M5k37Db9dT+mdHhk4L7zPWAUu7w2gUDXa7wknHNWzfjUeCLraNtvT # X4/edIhJEqGCAs0wggI2AgEBMIH4oYHQpIHNMIHKMQswCQYDVQQGEwJVUzETMBEG # A1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWlj # cm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBP # cGVyYXRpb25zMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjpFNUE2LUUyN0MtNTky # RTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZaIjCgEBMAcG # BSsOAwIaAxUAq6fBtEENocNASMqL03zGJS0wZd2ggYMwgYCkfjB8MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQg # VGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIFAOU9VGowIhgPMjAy # MTExMTYwNTQ5MzBaGA8yMDIxMTExNzA1NDkzMFowdjA8BgorBgEEAYRZCgQBMS4w # LDAKAgUA5T1UagIBADAJAgEAAgEAAgH/MAcCAQACAhE/MAoCBQDlPqXqAgEAMDYG # CisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEA # AgMBhqAwDQYJKoZIhvcNAQEFBQADgYEAX5ibpWzGI7j/LE37sRafqryVMZ8R328V # g3agiEkJqkvKPbBxL4XypOwg/9cPW7AVThob1AFy3X23M9fgD9wjh/vnRls5uB26 # G46OAJTwTZoNCew1CxWINtZ8PgTvqgcuuETEsEi9l22wDzmtq1FUqNy8kzwM57O4 # 2q248sd+qW8xggMNMIIDCQIBATCBkzB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0Eg # MjAxMAITMwAAAUedj/Hm3jGDWQAAAAABRzANBglghkgBZQMEAgEFAKCCAUowGgYJ # KoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJBDEiBCBc8Ie4PSt7 # DR9CmppgiXaWdy8SJLIBV1CsNRWkgSACnTCB+gYLKoZIhvcNAQkQAi8xgeowgecw # geQwgb0EIHvbPBIDlM+6BsiJk7/YfWGuKwBUi3DMOxxvRaqKGOmFMIGYMIGApH4w # fDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMd # TWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAFHnY/x5t4xg1kAAAAA # AUcwIgQgajLxHBQSOf9M7tB3AFxNTyY3Eyarxng0k+IY6GKYiZowDQYJKoZIhvcN # AQELBQAEggEAWVgGCZy5LZSIHd38cfks7kgN+x00kRlHCCs5zojhYCijGkFD/UjR # stsLjk8rcweNUCZ8cg6+JAl8pY58++xBoaiPzwOyiAkQGcELG0MRAIJLOVvzf+BI # TgYNKCuadmsWx+6PAr1Vi6FhtbYeoPBQ99aLLwlEV5idx4Wn/VBB8RCw344xvFUi # 9Q3DKMMBsbzF/BcHOeDpnvxJnL/64ugRz5LV4g6b+3Q3SejivHPzEmzip61J8dZC # btxKAVlAnKKP4zHrKNDJgKmMQybJnbJXi+Qw0fsfa76m6OfOXQYCoSp+IAgC+02Y # XFH/gE3x1DuvEhzqsfN4B/CfFZ86qjVBmg== # SIG # End signature block |