AzStackHciConnectivity/AzStackHci.Connectivity.Helpers.psm1
class AzStackHciConnectivityTarget { # Attributes for Azure Monitor schema [string]$Name #Name of the individual test/rule/alert that was executed. Unique, not exposed to the customer. [string]$Title #User-facing name; one or more sentences indicating the direct issue. [string]$Severity #Severity of the result (Critical, Warning, Informational, Hidden) – this answers how important the result is. Critical is the only update-blocking severity. [string]$Description #Detailed overview of the issue and what impact the issue has on the stamp. [psobject]$Tags #Key-value pairs that allow grouping/filtering individual tests. For example, "Group": "ReadinessChecks", "UpdateType": "ClusterAware" [string]$Status #The status of the check running (i.e. Failed, Succeeded, In Progress) – this answers whether the check ran, and passed or failed. [string]$Remediation #Set of steps that can be taken to resolve the issue found. [string]$TargetResourceID #The unique identifier for the affected resource (such as a node or drive). [string]$TargetResourceName #The name of the affected resource. [string]$TargetResourceType #The type of resource being referred to (well-known set of nouns in infrastructure, aligning with Monitoring). [datetime]$Timestamp #The Time in which the HealthCheck was called. [psobject[]]$AdditionalData #Property bag of key value pairs for additional information. [string]$HealthCheckSource #The name of the services called for the HealthCheck (I.E. Test-AzureStack, Test-Cluster). # Attribute for performing check [string[]]$EndPoint [string[]]$Protocol # Additional Attributes for end user interaction [string[]]$Service # short cut property to Service from tags [string[]]$OperationType # short cut property to Operation Type from tags [string[]]$Group # short cut property to group from tags [bool]$Mandatory # short cut property to mandatory from tags [bool]$System # targets for system checks such as proxy traversal [string]$Region # Region of the target [bool]$ArcGateway # Support by Arc Gateway and hence does not need checking explicitly } class AzStackHciConnectivityManifest { [string]$Title [System.Version]$Version [PsObject[]]$Targets } Import-LocalizedData -BindingVariable lcTxt -FileName AzStackHci.Connectivity.Strings.psd1 function Get-AzStackHciConnectivityServiceName { <# .SYNOPSIS Retrieve Services from built target packs .DESCRIPTION Retrieve Services from built target packs .EXAMPLE PS C:\> Get-AzStackHciServices Explanation of what the example does .INPUTS Service .OUTPUTS PSObject .NOTES #> [CmdletBinding()] param ( [Parameter(Mandatory = $false)] [string[]] $Service, [Parameter(Mandatory = $false)] [switch] $IncludeSystem ) try { Get-AzStackHciConnectivityTarget -IncludeSystem:$IncludeSystem | Select-Object -ExpandProperty Service | Sort-Object | Get-Unique } catch { throw "Failed to get services names. Error: $($_.Exception.Message)" } } function Get-AzStackHciConnectivityOperationName { <# .SYNOPSIS Retrieve Operation Types from built target packs .DESCRIPTION Retrieve Operation Types from built target packs e.g. Deployment, Update, Secret Rotation. .EXAMPLE PS C:\> Get-AzStackHciConnectivityOperationName Explanation of what the example does .INPUTS Service .OUTPUTS PSObject .NOTES #> [CmdletBinding()] param ( [Parameter(Mandatory = $false)] [string] $OperationType ) try { Get-AzStackHciConnectivityTarget | Select-Object -ExpandProperty OperationType | Sort-Object | Get-Unique } catch { throw "Failed to get services names. Error: $($_.Exception.Message)" } } function Get-AzStackHciConnectivityTarget { <# .SYNOPSIS Retrieve Endpoints from built target packs .DESCRIPTION Retrieve Endpoints from built target packs .EXAMPLE PS> Get-AzStackHciConnectivityTarget Get all connectivity targets .EXAMPLE Get-AzStackHciConnectivityTarget -Service ARC | ft Name, Title, Service, OperationType -AutoSize Get all ARC connectivity targets .EXAMPLE PS> Get-AzStackHciConnectivityTarget -Service ARC -OperationType Workload | ft Name, Title, Service, OperationType -AutoSize Get all ARC targets for workloads .EXAMPLE PS> Get-AzStackHciConnectivityTarget -OperationType Workload | ft Name, Title, Service, OperationType -AutoSize Get all targets for workloads .EXAMPLE PS> Get-AzStackHciConnectivityTarget -OperationType ARC -OperationType Update -Additive | ft Name, Title, Service, OperationType -AutoSize Get all ARC targets and all targets for Update .INPUTS Service - String array OperationType - String array Additive - Switch .OUTPUTS PSObject .NOTES #> [CmdletBinding()] param ( [Parameter(Mandatory = $false)] [string[]] $Service, [Parameter(Mandatory = $false)] [string[]] $OperationType, [Parameter(Mandatory = $false)] [switch] $Additive, [Parameter(Mandatory = $false)] [switch] $IncludeSystem, [Parameter(Mandatory = $false)] [switch] $LocalOnly, [Parameter()] [system.uri] $Uri = 'https://aka.ms/hciconnectivitytargets', [Parameter()] [psobject[]] $RuntimeConnectivityTarget, [Parameter()] [string] $RegionName, [Parameter()] [switch] $ARCGateway ) try { Import-AzStackHciConnectivityTarget -LocalOnly:$LocalOnly -Uri $Uri -RuntimeConnectivityTarget $RuntimeConnectivityTarget -RegionName $RegionName $executionTargets = @() # Additive allows the user to "-OR" their parameter values if ($Additive) { Write-Verbose -Message "Getting targets additively" if (-not [string]::IsNullOrEmpty($Service)) { Write-Verbose -Message ("Getting targets by Service: {0}" -f ($Service -join ',')) foreach ($svc in $Service) { $executionTargets += $Script:AzStackHciConnectivityTargets | Where-Object { $svc -in $_.Service } } } if (-not [string]::IsNullOrEmpty($OperationType)) { Write-Verbose -Message ("Getting targets by Operation Type: {0}" -f ($OperationType -join ',')) foreach ($Op in $OperationType) { $executionTargets += $Script:AzStackHciConnectivityTargets | Where-Object { $Op -in $_.OperationType } } } if ([string]::IsNullOrEmpty($OperationType) -and [string]::IsNullOrEmpty($Service)) { $executionTargets += $Script:AzStackHciConnectivityTargets } } else { if ([string]::IsNullOrEmpty($OperationType) -and [string]::IsNullOrEmpty($Service)) { $executionTargets += $Script:AzStackHciConnectivityTargets } elseif (-not [string]::IsNullOrEmpty($Service) -and [string]::IsNullOrEmpty($OperationType)) { Write-Verbose -Message ("Getting targets by Service: {0}" -f ($Service -join ',')) foreach ($svc in $Service) { $executionTargets += $Script:AzStackHciConnectivityTargets | Where-Object { $svc -in $_.Service } } } elseif (-not [string]::IsNullOrEmpty($OperationType) -and [string]::IsNullOrEmpty($Service)) { Write-Verbose -Message ("Getting targets by Operation Type: {0}" -f ($OperationType -join ',')) foreach ($Op in $OperationType) { $executionTargets += $Script:AzStackHciConnectivityTargets | Where-Object { $Op -in $_.OperationType } } } else { Write-Verbose -Message ("Getting targets by Operation Type: {0} and Service: {1}" -f ($OperationType -join ','), ($Service -join ',')) $executionTargetsByOp = @() foreach ($Op in $OperationType) { $executionTargetsByOp += $Script:AzStackHciConnectivityTargets | Where-Object { $Op -in $_.OperationType } } foreach ($svc in $Service) { $executionTargets += $executionTargetsByOp | Where-Object { $svc -in $_.Service } } } } # Always add Mandatory targets $executionTargets += $Script:AzStackHciConnectivityTargets | Where-Object Mandatory | ForEach-Object { if ($PSITEM -notin $executionTargets) { $PSITEM } } # Check the local agent incase we haven't been passed the ARCGateway switch if (-not $ARCGateway) { $ARCGateway = Get-AzStackHciARCGatewaySetting } if ($ARCGateway) { # Log substractions $substractions = $executionTargets | Where-Object { $_.ARCGateway } Write-Verbose "Removing the following definitions due to ARCGateway support: $($substractions | Format-table Name, ARCGateway, EndPoint | Out-String)" # Any endpoint defined as an ARCGateway does not need to be checked explicitly $executionTargets = $executionTargets | Where-Object { !$_.ARCGateway } } # AzureLocal should only be AzureLocal # Fairfax should only be Fairfax # Regular regions should be region + global e.g. EastUS + Global if ($RegionName -eq 'AzureLocal') { $executionTargets = $executionTargets | Where-Object { $_.Region -eq 'AzureLocal' } } elseif ($RegionName -eq 'Fairfax') { $executionTargets = $executionTargets | Where-Object { $_.Region -eq 'Fairfax' } } else { $executionTargets = $executionTargets | Where-Object { $RegionName -in $_.Region -or 'Global' -in $_.Region } } if ($IncludeSystem) { return $executionTargets } else { return ($executionTargets | Where-Object Service -NotContains 'System') } } catch { throw "Get failed: $($_.exception)" } } function Validate-RuntimeConnectivityTarget { <# .SYNOPSIS Validate Runtime Connectivity Target .DESCRIPTION Validate Runtime Connectivity Target .EXAMPLE PS C:\> Validate-RuntimeConnectivityTarget Explanation of what the example does .INPUTS URI .OUTPUTS Output (if any) .NOTES #> [CmdletBinding()] param ( [Parameter()] [psobject[]] $RuntimeConnectivityTarget ) try { foreach ($target in $RuntimeConnectivityTarget) { # this maybe bolstered to include more checks as needed like valid hostname, valid protocol, etc. ##### QUESTION: should we only allow 1 endpoint and 1 protocol per custom rule? if (-not $target.DisplayName -or $target.DisplayName -isnot [string]) { throw $lcTxt.RuntimeConnectivityTargetFailed -f "DisplayName", 'a String' } if (-not $target.Description -or $target.Description -isnot [string]) { throw $lcTxt.RuntimeConnectivityTargetFailed -f "Description", 'a String' } if (-not $target.Service -or $target.Service -isnot [System.Array]) { throw $lcTxt.RuntimeConnectivityTargetFailed -f "Service", 'an Array' } if (-not $target.EndPoint -or $target.EndPoint -isnot [System.Array]) { throw $lcTxt.RuntimeConnectivityTargetFailed -f "EndPoint", 'an Array' } if (-not $target.Protocol -or $target.Protocol -notmatch 'http|https') { throw $lcTxt.RuntimeConnectivityTargetFailed -f "Protocol", 'http or https' } if (-not $target.Severity -or $target.Severity -notmatch 'CRITICAL|WARNING') { throw $lcTxt.RuntimeConnectivityTargetFailed -f "Severity", 'CRITICAL or WARNING' } if (-not $target.Remediation -or $target.Remediation -notmatch 'https?://.*') { throw $lcTxt.RuntimeConnectivityTargetFailed -f "Remediation", 'a URL' } } } catch { throw $_ } } function Validate-CustomDefinitionUri { <# .SYNOPSIS Validate Custom Definition Uri .DESCRIPTION Validate Custom Definition Uri .EXAMPLE PS C:\> Validate-CustomDefinitionUri -Uri $Uri Explanation of what the example does .INPUTS URI .OUTPUTS Output (if any) .NOTES #> [CmdletBinding()] param ( [Parameter()] [system.uri] $Uri ) try { if ($Uri.Scheme -ne 'https') { throw $lcTxt.CustomDefinitionUriFailed -f "Uri" } } catch { throw $_ } } function Import-AzStackHciConnectivityTarget { <# .SYNOPSIS Retrieve Endpoints from built target packs .DESCRIPTION Retrieve Endpoints from built target packs .EXAMPLE PS C:\> Import-AzStackHciConnectivityTarget Explanation of what the example does .INPUTS URI .OUTPUTS PSObject .NOTES #> [CmdletBinding()] param ( [switch] $LocalOnly, [Parameter()] [system.uri] $Uri = 'https://aka.ms/hciconnectivitytargets', [Parameter()] [psobject[]] $RuntimeConnectivityTarget, [Parameter()] [string] $RegionName ) try { $Script:AzStackHciConnectivityTargets = @() if (-not $LocalOnly) { Write-Verbose "Trying to get targets from: $Uri/$RegionName" $Script:AzStackHciConnectivityTargets += Get-CloudEndpointFromManifest -Uri "$Uri/$RegionName" } # if region specific targets are not found, fall back to global targets if ((-not $LocalOnly) -and (-not $Script:AzStackHciConnectivityTargets)) { Write-Verbose "Trying to get targets from: $Uri" $Script:AzStackHciConnectivityTargets += Get-CloudEndpointFromManifest -Uri $Uri } if ($Script:AzStackHciConnectivityTargets) { return } else { $targetFiles = Get-ChildItem -Path "$PSScriptRoot\Targets\*.json" | Select-Object -ExpandProperty FullName Write-Verbose ("Importing {0}" -f ($targetFiles -join ',')) ForEach ($targetFile in $targetFiles) { try { # TO DO - Add validations: # - protocol should not contain :// $targetPackContent = Get-Content -Path $targetFile | ConvertFrom-Json -WarningAction SilentlyContinue foreach ($target in $targetPackContent) { #Set Name of the individual test/rule/alert that was executed. Unique, not exposed to the customer. $target | Add-Member -MemberType NoteProperty -Name HealthCheckSource -Value $ENV:EnvChkrId $target.TargetResourceID = $target.EndPoint -join '_' $target.TargetResourceName = $target.EndPoint -join '_' $target.TargetResourceType = 'External Endpoint' $Script:AzStackHciConnectivityTargets += [AzStackHciConnectivityTarget]$target } } catch { throw ("Unable to read {0}. Error: {1}" -f (Split-Path -Path $targetFile -Leaf), $_.Exception.Message) } } # adding this here so it is subject to any filters (include/exclude/file-based override) that the user may have set ##### QUESTION: should this throw an exception if the target is malformed, or continue with the rest of the targets/tests? (currently throws exception) if ($RuntimeConnectivityTarget.count -ge 1) { foreach ($rtTarget in $RuntimeConnectivityTarget) { $Script:AzStackHciConnectivityTargets += ConvertTo-AzStackHciConnectivityTarget -RuntimeConnectivityTarget $rtTarget } } } } catch { throw "Import failed: $($_.exception)" } } function ConvertTo-AzStackHciConnectivityTarget { <# .SYNOPSIS Convert Runtime connectivity (psobject) target to AzStackHciConnectivityTarget .DESCRIPTION Convert Runtime connectivity (psobject) target to AzStackHciConnectivityTarget .EXAMPLE PS C:\> ConvertTo-AzStackHciConnectivityTarget -RuntimeConnectivityTarget $RuntimeConnectivityTarget .INPUTS URI .OUTPUTS Output (if any) .NOTES #> [CmdletBinding()] param ( [Parameter()] [psobject] $RuntimeConnectivityTarget ) try { Log-Info "Converting Runtime Connectivity Target to AzStackHciConnectivityTarget:" Log-Info ($RuntimeConnectivityTarget | Out-String) $hash = @{ Name = "AzStackHci_Connectivity_{0}_{1}" -f ($RuntimeConnectivityTarget.Service | Select-Object -first 1), $RuntimeConnectivityTarget.DisplayName -replace '\s', '_' Service = $RuntimeConnectivityTarget.Service Title = $RuntimeConnectivityTarget.DisplayName Severity = $RuntimeConnectivityTarget.Severity Description = $RuntimeConnectivityTarget.Description Remediation = $RuntimeConnectivityTarget.Remediation TargetResourceID = $RuntimeConnectivityTarget.EndPoint -join '_' -replace '\*', 'www' TargetResourceName = $RuntimeConnectivityTarget.EndPoint -join '_' -replace '\*', 'www' TargetResourceType = 'External Endpoint' HealthCheckSource = $ENV:EnvChkrId Protocol = $RuntimeConnectivityTarget.Protocol EndPoint = $RuntimeConnectivityTarget.EndPoint # Any runtime endpoint will need to be global and ARCGateway:false to ensure it is not filtered out Region = 'Global' ARCGateway = $false } $target = New-Object -TypeName AzStackHciConnectivityTarget -Property $hash return $target } catch { throw $_ } } function Get-SigningRootChain { <# .SYNOPSIS Get signing root for https endpoint .DESCRIPTION Get signing root for https endpoint .EXAMPLE PS C:\> Get-SigningRoot -uri MicrosoftOnline.com Explanation of what the example does .INPUTS URI .OUTPUTS Output (if any) .NOTES #> [CmdletBinding()] param ( [Parameter()] [System.Uri] $Uri, [Parameter()] [System.Management.Automation.Runspaces.PSSession] $PsSession, [Parameter()] [string] $Proxy, [Parameter()] [pscredential] $proxyCredential ) try { $sb = { $uri = $args[0] $proxy = $args[1] $proxyCredential = $args[2] $GetSslCertChainFunction = $args[3] if (-not (Get-Command -Name Get-SslCertificateChain -ErrorAction SilentlyContinue)) { throw "Cannot find Get-SslCertificateChain in AzStackHci.EnvironmentChecker.PortableUtilities module" } else { Write-Verbose "Found Get-SslCertificateChain in AzStackHci.EnvironmentChecker.Utilities module" $chain = Get-SslCertificateChain -Url $Uri -Proxy $Proxy -ProxyCredential $ProxyCredential } return $chain.ChainElements.Certificate } $ChainElements = if ($PsSession) { Invoke-Command -Session $PsSession -ScriptBlock $sb -ArgumentList $Uri, $Proxy, $ProxyCredential,${function:Get-SslCertificateChain} } else { Invoke-Command -ScriptBlock $sb -ArgumentList $Uri, $Proxy, $ProxyCredential,${function:Get-SslCertificateChain} } return $ChainElements } catch { throw $_ } } function Test-RootCA { <# .SYNOPSIS Short description .DESCRIPTION Long description .EXAMPLE PS C:\> <example usage> Explanation of what the example does .INPUTS Inputs (if any) .OUTPUTS Output (if any) .NOTES General notes #> [CmdletBinding()] param( [Parameter()] [System.Management.Automation.Runspaces.PSSession] $PsSession, [Parameter()] [string] $Proxy, [Parameter()] [pscredential] $ProxyCredential ) try { if ($Script:AzStackHciConnectivityTargets) { $rootCATarget = $Script:AzStackHciConnectivityTargets | Where-Object Name -EQ System_Check_SSL_Inspection_Detection if ($rootCATarget.count -ne 1) { throw "Expected 1 System_RootCA, found $($rootCATarget.count)" } if ($PsSession) { Copy-RemoteItem -PsSession $PsSession -SourcePath (Join-Path (Split-Path -Parent $PSScriptRoot) "AzStackHci.EnvironmentChecker.PortableUtilities.psm1") -CmdletName "Get-SslCertificateChain" } # We have two endpoints to check, they expire 6 months apart # meaning we should get a warning if criteria needs to change # 1 only require 1 endpoint to not be re-encrypted to succeed. $rootCATargetUrls = @() $rootCATarget.EndPoint | Foreach-Object { foreach ($p in $rootCATarget.Protocol) { $rootCATargetUrls += "{0}://{1}" -f $p,$PSITEM } } $AdditionalData = @() foreach ($rootCATargetUrl in $rootCATargetUrls) { Log-Info "Testing SSL chain for $rootCATargetUrl" [array]$ChainElements = Get-SigningRootChain -Uri $rootCATargetUrl -PsSession $PsSession -Proxy $Proxy -ProxyCredential $ProxyCredential # This is our canary internet endpoint, if we can't get the chain we probably don't have internet access. if ($null -eq $ChainElements) { $Status = 'FAILURE' $detail = "Failed to get certificate chain for $rootCATargetUrl. Ensure the endpoint is accessible and proxy configuration is correct." Log-Info $detail -Type Warning } else { # Remove the leaf as this will always contain O=Microsoft in its subject $ChainElements = $ChainElements[1..($ChainElements.Length-1)] $subjectMatchCount = 0 # We check for 2 expected subjects and only require 1 to succeed $rootCATarget.Tags.ExpectedSubject | Foreach-Object { if ($ChainElements.Subject -match $PSITEM) { $subjectMatchCount++ } } if ($subjectMatchCount -ge 1) { $Status = 'SUCCESS' $detail = "Expected at least 1 chain certificate subject to match $($rootCATarget.Tags.ExpectedSubject -join ' or '). $subjectMatchCount matched." Log-Info $detail } else { $Status = 'FAILURE' $detail = "Expected at least 1 chain certificate subjects to match $($rootCATarget.Tags.ExpectedSubject -join ' or '). $subjectMatchCount matched. Actual subjects $($ChainElements.Subject -join ','). SSL decryption and re-encryption detected." Log-Info $detail -Type Error } } $AdditionalData += @{ Source = if ([string]::IsNullOrEmpty($PsSession.ComputerName)) { $ENV:COMPUTERNAME } else { $PsSession.ComputerName } Resource = $rootCATargetUrl Status = $Status Detail = $detail TimeStamp = [datetime]::UtcNow } } $result = @() $result += $AdditionalData | ForEach-Object { $params = @{ Name = $rootCATarget.Name Title = $rootCATarget.Title DisplayName = $rootCATarget.Title Severity = $rootCATarget.Severity Description = $rootCATarget.Description Tags = @{ Service = 'System' Mandatory = $true } Remediation = 'https://learn.microsoft.com/en-us/azure-stack/hci/deploy/deployment-tool-checklist' TargetResourceID = "$($PsItem.Source)/$($PsItem.Resource)" TargetResourceName = $PsItem.Resource TargetResourceType = $rootCATarget.TargetResourceType Timestamp = $PsItem.TimeStamp Status = $PsItem.Status AdditionalData = $PsItem HealthCheckSource = $ENV:EnvChkrId } New-AzStackHciResultObject @params } return $result } else { throw "No AzStackHciConnectivityTargets" } } catch { Log-Info "Test-RootCA failed with error: $($_.exception.message)" -Type Warning } } function Get-UniqueEndpoint { [CmdletBinding()] param ( [Parameter()] [PsObject] $Target ) # Build list of unique endpoints $uriList = @() foreach ($t in $Target) { foreach ($u in $t.EndPoint) { foreach ($p in $t.Protocol) { $uriList += "{0}://{1}" -f $p.tolower(), ($u -Replace '\*', 'www') } } } Log-Info "Uri Count (total): $($uriList.count)" $uriList = $uriList | Sort-Object -Unique Log-Info "Uri Count (Unique): $($uriList.count)" return $uriList } function Invoke-WebRequestEx { <# .SYNOPSIS Get Connectivity via Invoke-WebRequest .DESCRIPTION Get Connectivity via Invoke-WebRequest, supporting proxy. This function takes a connectivity target definition, creates a PS(5) Job for each endpoint and protocol and returns the results. If PsSession is provided, the jobs are run on the remote machine. Success is defined as a 200 status code or a valid status code (not service available), with a GET method and the response Uri is the same as the request Uri. .EXAMPLE PS C:\> Invoke-WebRequestEx -Target $Target Explanation of what the example does .INPUTS URI .OUTPUTS Output (if any) .NOTES In the case of a proxy being provided, the proxy is used for all endpoints. In the case of a proxy being configured on the box, invoke-webrequest will use the wininet proxy for all calls. Certificate Validation is disabled for the calls. #> [CmdletBinding()] param ( [Parameter()] [psobject[]] $Target, [Parameter()] [System.Management.Automation.Runspaces.PSSession[]] $PsSession, [Parameter()] [string] $Proxy, [Parameter()] [pscredential] $ProxyCredential ) if (-not $Target) { Log-Info "No targets to test. Returning No_Target result" $params = @{ Name = 'AzStackHci_Connectivity_No_Targets' Title = 'No Connectivity Targets to Test' DisplayName = 'No Connectivity Targets to Test' Severity = 'INFORMATIONAL' Description = 'No Connectivity Targets to Test' Tags = @{ Service = $_.Service Mandatory = $_.Mandatory ARCGateway = $_.ARCGateway Region = $_.Region } Remediation = 'No action required' TargetResourceID = $PsSession -join ',' TargetResourceName = $PsSession -join ',' TargetResourceType = 'Endpoint' Timestamp = [datetime]::UtcNow Status = 'SUCCESS' AdditionalData = @{} HealthCheckSource = $ENV:EnvChkrId } return (New-AzStackHciResultObject @params) } $Target | Add-Member -MemberType NoteProperty -Name TimeStamp -Value [datetime]::UtcNow -Force $ScriptBlock = { $Uri = $args[0] $TimeoutSecs = $args[1] $maxJobs = $args[2] $Proxy = $args[3] $ProxyCredential = $args[4] $timeoutSecondsDefault = 10 if ([string]::IsNullOrEmpty($TimeoutSecs)) { $timeout = $timeoutSecondsDefault } else { $timeout = $TimeoutSecs } # Create an array of jobs for all uris and protocols $inProgressJobList = @() $totalJobList = @() $results = @() foreach ($u in $uri) { $iwrScriptBlock = { # Define function for tracing redirects to enhance diagnostics function Trace-Redirects { param ( [Parameter(Mandatory = $true)] [hashtable]$iwrSplat ) # Run invoke-webrequest while headers are being redirected $traceLog = "" $iwrSplat.Add('MaximumRedirection', 0) $maximumRedirectsCount = 5 # guard against too many redirects $redirectCount = 0 do { $redirectCount++ try { $traceLog += "Testing {0}... " -f $iwrSplat.Uri $traceOutput = Invoke-WebRequest @iwrSplat -ea SilentlyContinue if ($traceOutput.StatusCode -ge 300 -and $traceOutput.StatusCode -lt 400) { if ($traceOutput.Headers.Location -ne $null) { $traceLog += "Redirected ({0})`r`n" -f $traceOutput.StatusCode $iwrSplat.Uri = $traceOutput.Headers.Location } else { $traceLog += "Headers Location Empty. Not expected" break } } else { $traceLog += ("StatusCode: {0}`r`n" -f $traceOutput.StatusCode) break } } catch { $traceLog += "Exception: {0}" -f $_.Exception.Message break } } while (($traceOutput.StatusCode -ge 300 -and $traceOutput.StatusCode -lt 400) -or $redirectCount -lt $maximumRedirectsCount) # only return traceLog if we have a redirect else the original response/exception is sufficient if ($redirectCount -gt 1) { return $traceLog } else { return $null } } # Test endpoint $retry = 0 $maxRetry = 3 do { $retry++ if ($retry -gt 1) { Start-Sleep -Seconds 5 } try { $uri = $args[0] $proxy = $args[1] $proxyCred = $args[2] $Timeout = $args[3] $iwrParams =@{ Uri = $Uri UseBasicParsing = $true TimeoutSec = 30 } # Ignore certificate validation and use TLS 1.2 if (-not ([System.Management.Automation.PSTypeName]'ServerCertificateValidationCallback').Type) { $certCallback = @" using System; using System.Net; using System.Net.Security; using System.Security.Cryptography.X509Certificates; public class ServerCertificateValidationCallback { public static void Ignore() { if(ServicePointManager.ServerCertificateValidationCallback == null) { ServicePointManager.ServerCertificateValidationCallback += delegate ( Object obj, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors ) { return true; }; } } } "@ $null = Add-Type $certCallback $null = [ServerCertificateValidationCallback]::Ignore() $null = [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 } if ($proxy) { $iwrParams.Add('Proxy', $proxy) $iwrParams.Add('ProxyUseDefaultCredentials', $true) } if ($proxyCred) { $iwrParams.Remove('ProxyUseDefaultCredentials') $iwrParams.Add('ProxyCredential', $proxyCred) } $stopwatch = [System.Diagnostics.Stopwatch]::new() $Stopwatch.Start() $webOut = Invoke-WebRequest @iwrParams $Stopwatch.Stop() $statusCode = $webout.StatusCode $webResponse = $webOut.BaseResponse $headers = $webOut.Headers $content = $webOut.Content } catch { if ($stopwatch.IsRunning) { $stopwatch.Stop() } $webResponse = $_.Exception.Response if ($webResponse) { try { $statusCode = [int]$webResponse.StatusCode $headers = @{} $content = [System.Text.Encoding]::UTF8.GetString($webResponse.GetResponseStream().ToArray()) foreach ($header in $webResponse.Headers) { $headers.$header = $webResponse.GetResponseHeader($header) } if ($webResponse.ContentType -eq 'application/json') { $content = ConvertFrom-Json -InputObject $content } } catch {} } $exception = @{ExceptionMessage = $_.Exception.Message; ErrorDetails = $_.ErrorDetails.Message; NonHTTPFailure = [System.String]::IsNullOrEmpty($webResponse) } } # Response Analysis # if status code is 200 return true # otherwise if the status code is a HTTP status code and the response Uri is the same as the request Uri and the method is GET, return true $isHttpStatusCode = $webResponse.StatusCode -is [System.Net.HttpStatusCode] $responseUriMatch = ([system.uri]$webResponse.ResponseUri).Host -eq ([system.uri]$uri).Host $responseMethodIsGet = $webResponse.Method -eq 'GET' $serviceUnavailable = $webResponse.StatusCode -eq [System.Net.HttpStatusCode]::ServiceUnavailable $test = $webResponse.StatusCode -eq [System.Net.HttpStatusCode]::OK -or ($isHttpStatusCode -and $responseUriMatch -and $responseMethodIsGet -and !$serviceUnavailable) # Initialize some strings to feedback to the user in the detail field. Debug field still remains for troubleshooting with log. $Detail = "" $Detail += "Test Analysis - Overall Result: $test`r`n" $WebResponseDetail = "" $TestAnalysisDetail = "" $CertificateChainDetail = "" $RedirectAnalysisDetail = "" $ExceptionDetail = "" # Gather TCP net connection to test connectivity if it failed. if (-not $test -and $retry -eq $maxRetry) { $tnc = Test-NetConnection -ComputerName ([system.uri]$uri).Host -Port ([system.uri]$uri).Port -InformationLevel Quiet -WarningAction SilentlyContinue $TestAnalysisDetail += "Test Analysis - Layer 3 (tnc): $tnc`r`n" } # We will try to get the certificate chain and redirect analysis $TestAnalysisDetail += "Test Analysis - Exception Message: $($exception.ExceptionMessage)`r`n" try { Import-Module AzStackHci.EnvironmentChecker -ErrorAction SilentlyContinue $CertificateChainOutput = Get-SslCertificateChain -url $Uri | Select-Object -ExpandProperty ChainElements | Select-Object -Expand Certificate | ForEach-Object { $_.Subject } | Out-String $CertificateChainDetail += $CertificateChainOutput } catch { $CertificateChainDetail += "Failed to get certificate chain for $Uri. Error: $($_.Exception.Message)`r`n" } try { $RedirectOutput = Trace-Redirects -iwrSplat $iwrParams | Out-String $RedirectAnalysisDetail += $RedirectOutput } catch { $RedirectAnalysisDetail += "Failed to get redirect analysis for $Uri. Error: $($_.Exception.Message)`r`n" } # web response detail if ($isHttpStatusCode) { $WebResponseDetail += "`r`nWeb Response Data - StatusCode: $statusCode`r`n" $WebResponseDetail += "Web Response Data - RequestUri: $uri`r`n" $WebResponseDetail += "Web Response Data - ResponseUri: $($webResponse.ResponseUri)`r`n" $WebResponseDetail += "Web Response Data - Method: $($webResponse.Method)`r`n" $WebResponseDetail += "Web Response Data - Server: $($webResponse.Server)`r`n" $WebResponseDetail += "`r`nWeb Response Analysis - ServiceUnavailable: $serviceUnavailable`r`n" $WebResponseDetail += "Web Response Analysis - ResponseUriHostMatch: $responseUriMatch`r`n" $WebResponseDetail += "Web Response Analysis - ResponseMethodIsGet: $responseMethodIsGet`r`n" } else { $WebResponseDetail += "Web Response Analysis - Not Applicable, no web response received`r`n" } # ExceptionDetail if ($exception) { $ExceptionDetail += "Exception Data - ExceptionMessage: $($exception.ExceptionMessage)`r`n" } # Add all the details to the final output $Detail += $TestAnalysisDetail $Detail += "$WebResponseDetail`r`n" $Detail += "TLS Data:`r`n$CertificateChainDetail`r`n" $Detail += "Redirect Data:`r`n$RedirectAnalysisDetail`r`n" $Detail += "`r`n$ExceptionDetail`r`n" } while ($test -eq $false -and $retry -lt $maxRetry) return @{ Source = $env:ComputerName Retry = "$retry / $maxRetry" LatencyInMs = $stopwatch.ElapsedMilliseconds ExceptionMessage = $Exception.ExceptionMessage Resource = $Uri Protocol = $p Status = if ($test) { "SUCCESS" } else { "FAILURE" } TimeStamp = [datetime]::UtcNow StatusCode = $StatusCode Detail = $Detail DebugDtls = @{ 'Test' = $test 'Retries' = "$retry / $maxRetry" 'Uri' = $Uri 'StatusCode'= $statusCode 'TCPNetConnection' = $tnc 'WebResponse' = $webResponse 'Headers' = $headers 'Exception' = $exception 'serviceUnavailable' = $serviceUnavailable 'ResponseUriMatch' = $responseUriMatch 'ResponseMethodIsGet' = $responseMethodIsGet 'ResponseUri' = $webResponse.ResponseUri 'ExceptionMessage' = $Exception.ExceptionMessage 'ErrorDetails' = $Exception.ErrorDetails 'NonHTTPFailure' = $Exception.NonHTTPFailure 'Server' = $webResponse.Server 'PowerShellVersion' = $PSVersionTable.PSVersion.Major 'LatencyInMs' = $stopwatch.ElapsedMilliseconds } } } # Start job for a url # add job to inProgressJobList and totalJobList # check inProgressJobList is greater or equal than maxJobs # if so wait for any job to finish before adding more $job = Start-Job -PSVersion 5.1 -ArgumentList $u, $proxy, $proxyCred, $timeout -ScriptBlock $iwrScriptBlock $inProgressJobList += $job $totalJobList += $job.id if ($inProgressJobList.Count -ge $maxJobs) { $finishedJob = @() $finishedJob = $inProgressJobList | Wait-Job -Any if ($finishedJob) { $inProgressJobList = $inProgressJobList | Where-Object { $_ -ne $finishedJob } } } } Wait-Job -Id $totalJobList | Out-Null $results += Receive-Job -Id $totalJobList Remove-Job -Id $totalJobList return $results } # Set max concurrent jobs if ($ENV:EnvChkrId -like 'PreUpdate*') { $maxJobs = 3 } else { $maxJobs = 10 } # Create a copy of the Target object $result = $Target | Select-Object -Property * $UriList = Get-UniqueEndpoint -Target $Target $sessionArgs = @() if ($result) { $sessionArgs += @($uriList,$result.Tags.TimeoutSecs, $maxJobs) } if ($Proxy) { $sessionArgs += $Proxy } if ($ProxyCredential) { $sessionArgs += $ProxyCredential } # Run Invoke-WebRequests on remote machines if PsSession is provided $AdditionalDataResults = @() $AdditionalDataResults += if ($PsSession) { Log-Info "Sending requests to $($PsSession.ComputerName -join ',') to test $($UriList.Count) URIs, this may take a while." Invoke-Command -Session $PsSession -ScriptBlock $ScriptBlock -ArgumentList $sessionArgs Log-Info "Received responses from $($PsSession.ComputerName -join ',')." } else { Log-Info "Sending requests to $ENV:COMPUTERNAME to test $($UriList.Count) URIs, this may take a while." Invoke-Command -ScriptBlock $ScriptBlock -ArgumentList $sessionArgs Log-Info "Received responses from $ENV:COMPUTERNAME." } $finalResult = Update-TargetWithResult -Target $Target -AdditionalDataResults $AdditionalDataResults return $finalResult } function Log-EndPointResult { [CmdletBinding()] param ( [Parameter()] [psobject] $Result, [string] $Name, [string] $Severity ) # In the case of failures, log the debug info to aid troubleshooting if ( $Result.Status -eq 'FAILURE' ){ Log-Info ("{0}: {1} {2} ({3})" -f $Result.Status, $Result.Source, $Result.Resource, $result.ExceptionMessage) -Type $Severity -Function 'Invoke-WebrequestEx' Log-Info ("Debug {0}: {1} {2}" -f $Result.Source, $Result.Resource, ($Result.DebugDtls | ConvertTo-Json)) -Type $Severity -Function 'Invoke-WebrequestEx' } else { Log-Info ("{0}: {1} {2} ({3}ms)" -f $Result.Status, $Result.Source, $Result.Resource, $result.LatencyInMs) -Function 'Invoke-WebrequestEx' } } function Update-TargetWithResult { [CmdletBinding()] param ( [Parameter()] [PsObject[]] $Target, [Parameter()] [PsObject[]] $AdditionalDataResults ) # Match the test result to all the target that match the endpoint $result = $Target | Select-Object -Property * $finalResult = @() # Inspect each result foreach ($dataResult in $AdditionalDataResults) { # find the target that matches the endpoint $matchingResult = @() $matchingResult = $result | Where-Object { ([system.uri]$dataResult.Resource).Scheme -in $_.Protocol -and ($dataResult.Resource -replace "https?://", "") -in $_.EndPoint} $matchingResult | Foreach-Object { Log-EndPointResult -Result $dataResult -Severity $PsItem.Severity -Name $PsItem.Name $dataResult.DebugDtls = 'redacted' # Create output $params = @{ Name = $PsItem.Name Title = $PsItem.Title DisplayName = $PsItem.Title Severity = $PsItem.Severity Description = $PsItem.Description Tags = @{ Service = $_.Service Mandatory = $_.Mandatory ARCGateway = $_.ARCGateway Region = $_.Region } Remediation = $PsItem.Remediation TargetResourceID = "$($dataResult.Source)/$($dataResult.Resource)" TargetResourceName = $dataResult.Resource TargetResourceType = $PsItem.TargetResourceType Timestamp = $dataResult.TimeStamp Status = $dataResult.Status AdditionalData = $dataResult HealthCheckSource = $ENV:EnvChkrId } $finalResult += New-AzStackHciResultObject @params } } return $finalResult } function Get-ProxyDiagnostics { [CmdletBinding()] param( [Parameter()] [System.Management.Automation.Runspaces.PSSession[]] $PsSession, [Parameter()] [switch] $ARCGateway ) Log-Info "Gathering proxy diagnostics" $proxyConfigs = @() $proxyConfigsRecommendations = @() $proxyConfigConsistency = @() if ($PsSession) { foreach ($session in $PsSession) { $proxyConfigs += Get-WinHttpProxyHelper -PsSession $session $proxyConfigs += Get-ProxyEnvironmentVariable -PsSession $session } } else { $proxyConfigs += Get-WinHttpProxyHelper $proxyConfigs += Get-ProxyEnvironmentVariable } $proxyConfigConsistency += Test-ProxySettingsConsistency -ProxyConfig $proxyConfigs -ARCGateway:$ARCGateway return ($proxyConfigs + $proxyConfigConsistency + $proxyConfigsRecommendations) } function Get-WinHttpProxyHelper { [CmdletBinding()] param( [Parameter()] [System.Management.Automation.Runspaces.PSSession] $PsSession ) Log-Info "Gathering WinHttp Proxy settings" $proxyHelperSb = { # gather machine scope proxy settings try { $line1, $line2, $line3, $JsonLines = netsh winhttp show advproxy $machineScope = $JsonLines #ConvertFrom-Json (-Join $JsonLines) } catch { } # Make sure we return the same object type as the other proxy functions $AdditionalData = @() $AdditionalData += @{ Detail = $machineScope Source = $ENV:COMPUTERNAME Resource = 'WinHttp' Status = 'SUCCESS' TimeStamp = [datetime]::UtcNow } return $AdditionalData } $winHttpOutput = if ($PsSession) { Invoke-Command -Session $PsSession -ScriptBlock $proxyHelperSb $TargetResourceName = "WinHttp_Proxy_$($PsSession.ComputerName)" } else { Invoke-Command -ScriptBlock $proxyHelperSb $TargetResourceName = "WinHttp_Proxy_$($ENV:COMPUTERNAME)" } # Write our findings to the log and create output objects $results = @() $results += foreach ($AdditionalData in $winHttpOutput | Sort-Object Source) { Log-Info "Machine Scope Proxy for $($AdditionalData.Source) (netsh winhttp show advproxy):" $AdditionalData.Detail | Format-List | Out-String -Stream | ForEach-Object {if (![string]::IsNullOrEmpty($_)){ Log-Info $_}} # Create output object $params = @{ Name = 'AzStackHci_Connectivity_Collect_Proxy_Diagnostics_WinHttp' Title = 'WinHttp Proxy Settings' DisplayName = "$($AdditionalData.Source) WinHttp Proxy Settings" Severity = 'INFORMATIONAL' Description = 'Collects proxy configuration for WinHttp' Tags = @{ Service = 'System' } Remediation = "https://learn.microsoft.com/en-us/azure-stack/hci/manage/configure-proxy-settings#configure-proxy-settings-for-azure-stack-hci-operating-system" TargetResourceID = "$($AdditionalData.Source)/$($AdditionalData.Resource)" TargetResourceName = $AdditionalData.Resource TargetResourceType = 'Proxy Settings' Timestamp = [datetime]::UtcNow Status = $AdditionalData.Status AdditionalData = $AdditionalData HealthCheckSource = $ENV:EnvChkrId } New-AzStackHciResultObject @params } return $results } function Get-ProxyEnvironmentVariable { <# .SYNOPSIS Get Proxy configuration from environment variables .DESCRIPTION Get Proxy configuration from environment variables .EXAMPLE PS C:\> Get-ProxyEnvironmentVariable Explanation of what the example does .INPUTS URI .OUTPUTS Output (if any) .NOTES #> [CmdletBinding()] param ( [Parameter()] [System.Management.Automation.Runspaces.PSSession] $PsSession ) Log-Info "Gathering machine Proxy settings from environment variables" $envProxySb = { $AdditionalData = @() $AdditionalData += @{ Detail = @{ ProxyIsEnabled = [bool]([Environment]::GetEnvironmentVariable("HTTPS_PROXY","MACHINE")) Proxy = [Environment]::GetEnvironmentVariable("HTTPS_PROXY","MACHINE") ProxyBypass = [Environment]::GetEnvironmentVariable("NO_PROXY","Machine") } | ConvertTo-Json Source = $ENV:COMPUTERNAME Resource = 'Environment' Status = 'SUCCESS' TimeStamp = [datetime]::UtcNow } return $AdditionalData } [array]$EnvironmentProxyOutput = if ($PsSession) { Invoke-Command -Session $PsSession -ScriptBlock $envProxySb $TargetResourceName = "Environment_Proxy_$($PsSession.ComputerName)" } else { Invoke-Command -ScriptBlock $envProxySb $TargetResourceName = "Environment_Proxy_$($ENV:COMPUTERNAME)" } # Write our findings to the log $results = @() $results += foreach ($AdditionalData in $EnvironmentProxyOutput | Sort-Object Source) { Log-Info "Environment Scope Proxy for $($AdditionalData.Source) (ENV:HTTPS_PROXY):" $AdditionalData.Detail | Format-List | Out-String -Stream | ForEach-Object {if (![string]::IsNullOrEmpty($_)){ Log-Info $_}} # Create output object $params = @{ Name = 'AzStackHci_Connectivity_Collect_Proxy_Diagnostics_Environment' Title = 'Environment Proxy Settings' DisplayName = "$($AdditionalData.Source) Environment Proxy Settings" Severity = 'Information' Description = 'Collects proxy configuration from environment variables' Tags = @{ Service = 'System' } Remediation = "https://learn.microsoft.com/en-us/azure-stack/hci/manage/configure-proxy-settings#configure-proxy-settings-for-azure-arc-enabled-servers" TargetResourceID = "$($AdditionalData.Source)/$($AdditionalData.Resource)" TargetResourceName = $AdditionalData.Resource TargetResourceType = 'Proxy Settings' Timestamp = $AdditionalData.TimeStamp Status = $AdditionalData.Status AdditionalData = $AdditionalData HealthCheckSource = $ENV:EnvChkrId } New-AzStackHciResultObject @params } return $results } function Test-ProxySettingsConsistency { [CmdletBinding()] param( [Parameter()] [System.Array] $ProxyConfig, [Parameter()] [switch] $ARCGateway ) Log-Info "Testing all proxy settings are consistent" # TO DO agree on desired proxy config for ARCGateway # return dummy result for now if ($ARCGateway) { $params = @{ Name = 'AzStackHci_Connectivity_Proxy_Settings_ARCGateway' Title = 'Proxy Settings ARCGateway' DisplayName = 'Dummy result for ARCGateway Proxy Settings' Severity = 'Informational' Description = 'Checks that all nodes are configured consistently for proxy settings in ARCGateway mode' Tags = @{ Service = 'System' } Remediation = "https://learn.microsoft.com/en-us/azure-stack/hci/manage/configure-proxy-settings" TargetResourceID = "ARCGateway_Proxy_Settings" TargetResourceName = "ARCGateway_Proxy_Settings" TargetResourceType = 'ARC Gateway Proxy Settings' Timestamp = [datetime]::UtcNow Status = $Status AdditionalData = @{ Source = 'None' Resource = 'ARCGateway Proxy Settings' Status = 'SUCCESS' Detail = 'Dummy result for ARCGateway Proxy Settings' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } return (New-AzStackHciResultObject @params) } $simplifiedProxyConfig = Get-SimpleProxyConfigObject -ProxyConfig $ProxyConfig if (Compare-PSObjectArray -Array $simplifiedProxyConfig -Property Proxy) { # Check all winHttp are configured $dtl = $lcTxt.ProxySettingsConsistencyPass Log-Info $dtl $status = 'SUCCESS' } else { $dtl = $lcTxt.ProxySettingsConsistencyFail -f ([string]($simplifiedProxyConfig | Format-Table | Out-String) -replace '\r\n','') Log-Info $dtl -Type 'Critical' $status = 'FAILURE' } $params = @{ Name = 'AzStackHci_Connectivity_Proxy_Settings_Consistency' Title = 'Proxy Settings Consistency' DisplayName = "$($simplifiedProxyConfig.Source -join ',') Environment Proxy Settings" Severity = 'Critical' Description = 'Checks that all nodes are configured consistently for proxy settings' Tags = @{ Service = 'System' } Remediation = "https://learn.microsoft.com/en-us/azure-stack/hci/manage/configure-proxy-settings" TargetResourceID = "$($simplifiedProxyConfig.Source -join ',')/Proxy_Settings" TargetResourceName = "Proxy_Settings_AllServers" TargetResourceType = 'Proxy Settings' Timestamp = [datetime]::UtcNow Status = $Status AdditionalData = @{ Source = $simplifiedProxyConfig.Source -join ',' Resource = 'Proxy Settings' Status = $status Detail = $dtl TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } [array]$result = New-AzStackHciResultObject @params $ByPassCheck = Test-ProxyByPassListRecommendations -ProxyConfig $simplifiedProxyConfig return @($result + $ByPassCheck) } function Get-SimpleProxyConfigObject { param( [Parameter()] [System.Array] $ProxyConfig ) try { $simplifiedProxyConfig = @() $simplifiedProxyConfig += foreach ($p in $ProxyConfig) { $ProxyConfigObject = ConvertFrom-Json -InputObject $p.AdditionalData.Detail $hash = @{ Source = $p.AdditionalData.Source Resource = $p.AdditionalData.Resource ProxyIsEnabled = $ProxyConfigObject.ProxyIsEnabled Proxy = $ProxyConfigObject.Proxy -replace 'http://|https://','' ProxyBypass = $ProxyConfigObject.ProxyBypass -replace ';',',' } New-Object -TypeName PsObject -Property $hash } return $simplifiedProxyConfig } catch { throw $_ } } function Test-ProxyByPassListRecommendations { [CmdletBinding()] param ( [Parameter()] [System.Array] $ProxyConfig ) if ($true -in $ProxyConfig.ProxyIsEnabled) { $requiredByPassList = @("localhost","127.0.0.1",".svc","10.0.0.0/8","172.16.0.0/12","192.168.0.0/16") # should also contain domain name, but we don't know that. Log-Info ($lcTxt.ProxyByPassListRecommendations -f ($requiredByPassList -join ',')) $result = @() foreach ($config in $ProxyConfig) { # Check if each entry is present $missingByPassEntry = @() foreach ($entry in $requiredByPassList) { if ($entry -notin ($config.ProxyBypass -split ',')) { $missingByPassEntry += $entry } } # Set status based on missing entries if ($missingByPassEntry.count -gt 0) { $ByPassDtl = ($lcTxt.ProxyByPassListRecommendationsMissing -f ($missingByPassEntry -join ','), $config.Resource, $config.Source) Log-Info $ByPassDtl -Type Warning $status = 'FAILURE' } else { $ByPassDtl = ($lcTxt.ProxyByPassListRecommendationsPass -f $config.Resource, $config.Source) Log-Info $ByPassDtl $status = 'SUCCESS' } $params = @{ Name = 'AzStackHci_Connectivity_Proxy_ByPassList_Recommendations' Title = 'Proxy ByPassList Recommendations' DisplayName = "$($config.Source) Proxy ByPassList Recommendations" Severity = 'WARNING' Description = 'Checks that all nodes are configured with recommended proxy bypass list items' Tags = @{ Service = 'System' } Remediation = "https://learn.microsoft.com/en-us/azure-stack/hci/manage/configure-proxy-settings" TargetResourceID = "$($config.Source)/Proxy_Bypass_List" TargetResourceName = $config.Resource TargetResourceType = 'Proxy Settings' Timestamp = [datetime]::UtcNow Status = $Status AdditionalData = @{ Source = $config.Source Resource = "Proxy_ByPassList_Recommendations_$($config.Source)" Status = $status Detail = $ByPassDtl TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $result += New-AzStackHciResultObject @params #region Critical Failure for proxy bypass invalid entries [System.String] $bypassCriticalRstStatus = "SUCCESS" [System.String] $bypassCriticalRstDetail = "" if ($config.ProxyBypass -match ",,") { $bypassCriticalRstStatus = "FAILURE" $bypassCriticalRstDetail += "Proxy bypass list contains ',,' in $($config.Resource) on $($config.Source)." } if (($config.Resource -eq "Environment") -and ($config.ProxyBypass.Contains("*"))) { $bypassCriticalRstStatus = "FAILURE" $bypassCriticalRstDetail += "Proxy bypass list from environment variable contains '*' in $($config.Resource) on $($config.Source)." } if ($config.ProxyBypass -match "<local>") { $bypassCriticalRstStatus = "FAILURE" $bypassCriticalRstDetail += "Proxy bypass list contains '<local>' in $($config.Resource) on $($config.Source)." } if ([System.String]::IsNullOrEmpty($bypassCriticalRstDetail)) { $bypassCriticalRstDetail = "Entries in proxy bypass list in $($config.Resource) on $($config.Source) are valid." } $proxyByPassInvalidEntriesCheckRstObject = @{ Name = 'AzStackHci_Connectivity_Proxy_ByPassList_InvalidEntries' Title = 'Proxy ByPassList Invalid Entries' DisplayName = "$($config.Source) Proxy Invalid Entries" Severity = 'CRITICAL' Description = 'Checks that all nodes proxy bypass list does not have invalid entries in it' Tags = @{ Service = 'System' } Remediation = "https://learn.microsoft.com/en-us/azure-stack/hci/manage/configure-proxy-settings" TargetResourceID = "$($config.Source)/Proxy_Bypass_List" TargetResourceName = $config.Resource TargetResourceType = 'Proxy Settings' Timestamp = [datetime]::UtcNow Status = $bypassCriticalRstStatus AdditionalData = @{ Source = $config.Source Resource = "Proxy_ByPassList_InvalidEntries_$($config.Source)" Detail = $bypassCriticalRstDetail Status = $bypassCriticalRstStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $result += New-AzStackHciResultObject @proxyByPassInvalidEntriesCheckRstObject #endregion } return $result } else { Log-Info "No proxy is configured. Skipping proxy bypass list recommendations" } } function Write-FailedUrls { [CmdletBinding()] param ( $result ) if (-not [string]::IsNullOrEmpty($Global:AzStackHciEnvironmentLogFile)) { $file = Join-Path -Path (Split-Path $Global:AzStackHciEnvironmentLogFile -Parent) -ChildPath FailedUrls.txt } $FailedUrls = ($result.AdditionalData | Where-Object Status -NE 'SUCCESS').Resource | Sort-Object -Unique if ($FailedUrls.count -gt 0) { Log-Info ("[Over]Writing {0} to {1}" -f ($FailedUrls -split ','), $file) $FailedUrls | Out-File $file -Force Log-Info "`nFailed Urls log: $file" -ConsoleOut } } function Select-AzStackHciConnectivityTarget { <# .SYNOPSIS Apply user exclusions to Connectivity Targets #> [CmdletBinding()] param ( [Parameter()] [psobject] $Targets, [Parameter()] [string[]] $Exclude, [Parameter()] [string] $FilePath = "$PSScriptRoot\..\ExcludeTests.txt" ) try { $returnList = @($Targets) if ($exclude) { Log-Info "Removing tests $($exclude -join ',')" $returnList = $returnList | Where-Object { $_.Service | Select-String -Pattern $exclude -NotMatch } } if ($returnList.count -eq 0) { throw "No tests to perform after filtering" } if (Test-Path -Path $FilePath) { $fileExclusion = Get-Content -Path $FilePath Log-Info "Reading exclusion file $FilePath" -ConsoleOut Log-Info "Applying file exclusions: $($fileExclusion -join ',')" -ConsoleOut $returnList = $returnList | Where-Object {( $_.Service | Select-String -Pattern $fileExclusion -NotMatch ) -and ( $_.endpoint | Select-String -Pattern $fileExclusion -NotMatch )} } Log-Info "Test list: $($returnList.Name -join ',')" if ($returnList.Count -eq 0) { Log-Info -Message "No tests to run." -ConsoleOut -Type Warning break noTestsBreak } return $returnList } catch { Log-Info "Failed to filter test list. Error: $($_.exception)" -Type Warning } } function Get-CloudEndpointFromManifest { [CmdletBinding()] param ( [Parameter()] [system.uri] $Uri ) try { $tempXmlFile = Join-Path -Path $env:temp -ChildPath 'AzStackHciConnectivityTarget.xml' Write-Verbose "Retrieving connectivity targets from $Uri to temp location: $tempXmlFile..." $iwrParams = @{ Uri = $Uri UseBasicParsing = $true OutFile = $tempXmlFile } $response = Invoke-WebRequest @iwrParams $testSigningScript = Join-Path -Path $PSScriptRoot -ChildPath 'TestXmlSigning.ps1' Write-Verbose "Validating signature of $tempXmlFile..." [bool]$checkSigningResult = Start-Job -PSVersion 5.1 -ScriptBlock { & $USING:testSigningScript $USING:tempXmlFile } | Wait-Job | Receive-Job -ErrorAction SilentlyContinue Write-Verbose "Signature validation result: $checkSigningResult" if (-not $checkSigningResult) { throw "Failed to validate signature of $tempXmlFile from $url" } [xml]$manifest = Get-Content -Path $tempXmlFile $version = $manifest.Objects.Object.Property | Where-Object Name -eq Version | Select -ExpandProperty '#text' $title = $manifest.Objects.Object.Property | Where-Object Name -eq Title | Select -ExpandProperty '#text' $targets = $manifest.Objects.Object.Property | Where-Object Name -eq Targets | Select -ExpandProperty 'Property' [AzStackHciConnectivityTarget[]]$targets = New-AzStackHciConnectivityTargetFromXml -TargetXml $targets if ($targets.count -eq 0) { throw "No connectivity targets found in $tempXmlFile" } # Make sure DNS and SSL tests are present if ('System_Check_DNS_External_Hostname_Resolution' -notin $targets.Name) { Log-Info 'System_Check_DNS_External_Hostname_Resolution is missing from the connectivity targets' -Type Warning } if ('System_Check_SSL_Inspection_Detection' -notin $targets.Name) { throw 'System_Check_SSL_Inspection_Detection is missing from the connectivity targets' } $msg = "Retrieved $($targets.count) connectivity targets from $($title) version $($version)" Write-Verbose $msg return $targets } catch { $msg = "Failed to get connectivity targets from $Uri. Error: $($_.exception.message)" Write-Verbose $msg } finally { if (Test-Path -Path $tempXmlFile) { Remove-Item -Path $tempXmlFile -ErrorAction SilentlyContinue } } } function Export-AzStackHciConnectivityTargetToXml { [CmdletBinding()] param ( [Parameter(Mandatory)] [string] $TargetDirectory, [Parameter(Mandatory)] [string] $TargetFileName, [Parameter(Mandatory)] [string] $Version, [string] $Region ) [array]$targetsForXml = Get-AzStackHciConnectivityTarget -LocalOnly -IncludeSystem -RegionName $Region if ($targetsForXml.Count -eq 0) { Write-Warning "No connectivity targets found for region $Region. Exiting..." return } Write-Host ("Found {0} connectivity targets for manifest" -f [int]($targetsForXml.Count)) Write-Host "Creating manifest with version $version" $manifest = New-Object AzStackHciConnectivityManifest -Property @{ Title = 'AzStackHci Connectivity Endpoint Definitions' Version = $Version Targets = $targetsForXml } $manifest | ConvertTo-Xml -Depth 5 -As Stream | Out-File $TargetDirectory\$TargetFileName -Encoding utf8 } function New-AzStackHciConnectivityTargetFromXml { param ($TargetXml) foreach ($target in $TargetXml) { $ErrorActionPreference = 'Stop' $AzStackHciConnectivityTargetObject = New-Object AzStackHciConnectivityTarget # Arrays 'EndPoint', 'Protocol', 'Service', 'OperationType' | Foreach-Object { $AzStackHciConnectivityTargetObject.$PSITEM = ($target.Property | Where-Object Name -eq $PSITEM).ChildNodes.'#text' } # Booleans 'Mandatory', 'System', 'ArcGateway' | ForEach-Object { $AzStackHciConnectivityTargetObject.$PSITEM = if (($target.Property | Where-Object Name -eq $PSITEM).'#text' -eq 'True') { $true } else { $false } } # Strings 'Name', 'Title', 'Severity', 'Description', 'Remediation', 'TargetResourceID', 'TargetResourceName', 'TargetResourceType', 'Group', 'Region' | Foreach-Object { $AzStackHciConnectivityTargetObject.$PSITEM = ($target.Property | Where-Object Name -eq $PSITEM).'#text' } # Tags $ErrorActionPreference = 'SilentlyContinue' $tagsProperties = $target.Property | Where-Object Name -eq Tags | Select-Object -ExpandProperty Property $Groups = $tagsProperties | Where-Object Name -eq Group $Mandatory = $tagsProperties | Where-Object Name -eq Mandatory $Service = $tagsProperties | Where-Object Name -eq Service $OperationType = $tagsProperties | Where-Object Name -eq OperationType $ExpectedSubject = $tagsProperties | Where-Object Name -eq ExpectedSubject $ArcGateway = $tagsProperties | Where-Object Name -eq ARCGateway # Tags are optional, we iterate through them to $tagHash = @{} if ($Groups) { $tagHash += @{Group = $Groups.'#text'}} if ($Service) { $tagHash += @{Service = $Service.ChildNodes.'#text'}} if ($Mandatory) { $tagHash += @{Mandatory = if ($Mandatory.'#text' -eq 'True') { $true } else { $false }}} if ($OperationType) { $tagHash += @{OperationType = $OperationType.ChildNodes.'#text'}} if ($ExpectedSubject) { $tagHash += @{ExpectedSubject = $ExpectedSubject.ChildNodes.'#text'}} if ($ArcGateway) { $tagHash += @{ARCGateway = if ($ArcGateway.'#text' -eq 'True') { $true } else { $false }}} $AzStackHciConnectivityTargetObject.Tags = New-Object PsObject -Property $tagHash $AzStackHciConnectivityTargetObject } } function Compare-PSObjectArray { param ( [Parameter(Mandatory=$true)] [PSObject[]]$Array, [Parameter(Mandatory=$true)] [string[]]$property ) # Get the first PSObject in the array $first = $Array[0] Log-Info "Comparing first object (below) objects with properties '$($property -join ',')':" Log-Info ($first | Sort-Object Source | Format-List | Out-String) # Compare the first PSObject to the rest of the PSObjects in the array $fail = @() for ($i = 1; $i -lt $Array.Count; $i++) { $result = Compare-Object $first $Array[$i] -Property $property # If the PSObjects do not match, return false if ($result) { $fail += $Array[$i] } } if ($fail.count -gt 0) { Log-Info "Objects properties '$($property -join ',')' do not match:" Log-Info ($fail | Sort-Object Source | Format-List | Out-String) return $false } # If all PSObjects match, return true return $true } # Convert PsObject to Hashtable function ConvertTo-Hashtable { param ( [Parameter(Mandatory=$true)] [PSObject]$Object ) $hash = @{} foreach ($property in $Object.PsObject.Properties) { $hash[$property.Name] = $property.Value } return $hash } function New-ARCGatewayRuntimeTarget { [CmdletBinding()] param ( [Parameter()] [string] $ARCGatewayName ) try { $properties = @{ Name = 'AzStackHci_Connectivity_ARCGateway_RuntimeTarget' DisplayName = 'ARC Gateway Runtime Target' Description = 'ARC Gateway Runtime Target' Service = @('ARCGateway') Endpoint = @("$ARCGatewayName.gw.arc.azure.net") Protocol = @('HTTPS') Severity = 'CRITICAL' # TO DO UPDATE THIS REMEDIATION LINK Remediation = 'https://aka.ms/ARC-Gateway-Connectivity-Requirements' ARCGateway = $false Region = 'Global' } return (New-Object -TypeName PSObject -Property $properties) } catch { throw "Failed to create ARC Gateway runtime target. Error: $($_.exception.message)" } } function Get-AzStackHciARCGatewaySetting { [CmdletBinding()] param() # attempt to detect if the user is using the ARCGateway scenario try { $ArcGateway = $false $azcmagent = "$env:ProgramW6432\AzureConnectedMachineAgent\azcmagent.exe" if (Test-Path $azcmagent) { Write-Verbose "Detected Azure Connected Machine Agent" $arcConnectionType = & $azcmagent config get connection.type Write-Verbose "ARC Gateway setting: $arcConnectionType" if ( $arcConnectionType -eq "gateway") { $ArcGateway = $true } } else { Write-Verbose "Azure Connected Machine Agent not detected" } } catch { Write-Verbose "Error checking ARC gateway settings: $($_.exception.message)" } return $ArcGateway } function Convert-EndpointsToAzureLocal { [CmdletBinding()] param ( [Parameter()] [string[]] $CloudFqdn ) $azureLocalDefaultFqdn = 'autonomous.cloud.private' $azureLocalEndpointsPath = Get-ChildItem -Path "$PSScriptRoot\Targets\*.json" -Include "*AzureLocal*" | Select-Object -ExpandProperty FullName try { (Get-Content -Path $azureLocalEndpointsPath).Replace($azureLocalDefaultFQDN, $CloudFqdn) | Set-Content -Path $azureLocalEndpointsPath } catch { throw "Failed to replace Default FQDN with AzureLocal FQDN. Error: $($_.exception.message)" } } # SIG # Begin signature block # MIIoUQYJKoZIhvcNAQcCoIIoQjCCKD4CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDvVouX94f6zW7x # U7FK2/3hGkMguNwKgASacdPBCHAbvKCCDYUwggYDMIID66ADAgECAhMzAAAEA73V # lV0POxitAAAAAAQDMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjQwOTEyMjAxMTEzWhcNMjUwOTExMjAxMTEzWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQCfdGddwIOnbRYUyg03O3iz19XXZPmuhEmW/5uyEN+8mgxl+HJGeLGBR8YButGV # LVK38RxcVcPYyFGQXcKcxgih4w4y4zJi3GvawLYHlsNExQwz+v0jgY/aejBS2EJY # oUhLVE+UzRihV8ooxoftsmKLb2xb7BoFS6UAo3Zz4afnOdqI7FGoi7g4vx/0MIdi # kwTn5N56TdIv3mwfkZCFmrsKpN0zR8HD8WYsvH3xKkG7u/xdqmhPPqMmnI2jOFw/ # /n2aL8W7i1Pasja8PnRXH/QaVH0M1nanL+LI9TsMb/enWfXOW65Gne5cqMN9Uofv # ENtdwwEmJ3bZrcI9u4LZAkujAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQU6m4qAkpz4641iK2irF8eWsSBcBkw # VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzUwMjkyNjAfBgNVHSMEGDAW # gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw # MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx # XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB # AFFo/6E4LX51IqFuoKvUsi80QytGI5ASQ9zsPpBa0z78hutiJd6w154JkcIx/f7r # EBK4NhD4DIFNfRiVdI7EacEs7OAS6QHF7Nt+eFRNOTtgHb9PExRy4EI/jnMwzQJV # NokTxu2WgHr/fBsWs6G9AcIgvHjWNN3qRSrhsgEdqHc0bRDUf8UILAdEZOMBvKLC # rmf+kJPEvPldgK7hFO/L9kmcVe67BnKejDKO73Sa56AJOhM7CkeATrJFxO9GLXos # oKvrwBvynxAg18W+pagTAkJefzneuWSmniTurPCUE2JnvW7DalvONDOtG01sIVAB # +ahO2wcUPa2Zm9AiDVBWTMz9XUoKMcvngi2oqbsDLhbK+pYrRUgRpNt0y1sxZsXO # raGRF8lM2cWvtEkV5UL+TQM1ppv5unDHkW8JS+QnfPbB8dZVRyRmMQ4aY/tx5x5+ # sX6semJ//FbiclSMxSI+zINu1jYerdUwuCi+P6p7SmQmClhDM+6Q+btE2FtpsU0W # +r6RdYFf/P+nK6j2otl9Nvr3tWLu+WXmz8MGM+18ynJ+lYbSmFWcAj7SYziAfT0s # IwlQRFkyC71tsIZUhBHtxPliGUu362lIO0Lpe0DOrg8lspnEWOkHnCT5JEnWCbzu # iVt8RX1IV07uIveNZuOBWLVCzWJjEGa+HhaEtavjy6i7MIIHejCCBWKgAwIBAgIK # YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm # aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw # OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD # VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG # 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la # UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc # 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D # dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ # lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk # kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 # A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd # X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL # 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd # sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 # T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS # 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI # bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL # BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD # uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF # BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h # cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA # YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn # 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 # v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b # pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ # KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy # CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp # mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi # hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb # BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS # oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL # gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX # cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGiIwghoeAgEBMIGVMH4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p # Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAAQDvdWVXQ87GK0AAAAA # BAMwDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEINzz # 3r4nu8JnmRolAdC2JQ8qeYV+SGdtnHZ88uUPSflZMEIGCisGAQQBgjcCAQwxNDAy # oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20wDQYJKoZIhvcNAQEBBQAEggEAAlHe3yJFWszMx4HxKY/SoKN+P4lZjy2Rmjf/ # zMqMlmaPYLQbQ6H30EpMNXS2YDoZVIPvTtvSWBh7XndT5kaLmfA7lxz1GAVCNf0T # yRsa3ehqLgicNVx8zeVZ7Ghio8rQnIBmTJl707RfbbdPRWUc5d0liJDzr1S43aXP # BTEOlEXE2erpaDSb6RmcrRc2uxqDv1YMKNzLRRd6htT+K2/04un+X6Qt/0GYw91m # id9qrnSSwL3cXDn5WuZxCK9oB49zJuOjk7LYN80Edak8og19DNu/VXdLSpTX2CB2 # R1Zn94/hbCw/2AZK4yvNcT+3LgYW1RBRH9M19cHrtoW0XfOohqGCF6wwgheoBgor # BgEEAYI3AwMBMYIXmDCCF5QGCSqGSIb3DQEHAqCCF4UwgheBAgEDMQ8wDQYJYIZI # AWUDBAIBBQAwggFZBgsqhkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGE # WQoDATAxMA0GCWCGSAFlAwQCAQUABCC2/X3p18ZgsfXIa0lSVywhY9WGGARF20v1 # 3a4w6u4ZqAIGaC397RUnGBIyMDI1MDYxMDE1NDgzMi41NFowBIACAfSggdmkgdYw # gdMxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsT # JE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJhdGlvbnMgTGltaXRlZDEnMCUGA1UECxMe # blNoaWVsZCBUU1MgRVNOOjQwMUEtMDVFMC1EOTQ3MSUwIwYDVQQDExxNaWNyb3Nv # ZnQgVGltZS1TdGFtcCBTZXJ2aWNloIIR+zCCBygwggUQoAMCAQICEzMAAAH+0KjC # ezQhCwEAAQAAAf4wDQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAg # UENBIDIwMTAwHhcNMjQwNzI1MTgzMTE4WhcNMjUxMDIyMTgzMTE4WjCB0zELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9z # b2Z0IElyZWxhbmQgT3BlcmF0aW9ucyBMaW1pdGVkMScwJQYDVQQLEx5uU2hpZWxk # IFRTUyBFU046NDAxQS0wNUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1l # LVN0YW1wIFNlcnZpY2UwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC8 # vCEXFaWoDjeiWwTg8J6BniZJ+wfZhNIoRi/wafffrYGZrJx1/lPe1DGk/c1abrZg # dSJ4hBfD7S7iqVrLgA3ciicj7js2mL1+jnbF0BcxfSkatzR6pbxFY3dt/Nc8q/Ts # 7XyLYeMPIu7LBjIoD0WZZt4+NqF/0zB3xCDKCQ+3AOtVAYvI6TIzdVOIcqqEa70E # IZVF0db2WY8yutSU9aJhX0tUIHlVh34ARS11+oB2qXNXEDncSDFKqGnolt8QqdN1 # x8/pPwyKvQevBNO1XaHbIMG2NdtAhqrJwo5vrfcZ9GSfbXos4MGDfs//HCGh1dPz # VkLZoc3t7EQOaZuJayyMa8UmSWLaDp23TV5KE6IaaFuievSpddwF6o1vpCgXyNf+ # 4NW1j2m8viPxoRZLj2EpQfSbOwK5wivBRL7Hwy5PS5/tVcIU0VuIJQ1FOh/EncHj # nh4YmEvR/BRNFuDIJukuAowoOIJG5vrkOFp4O9QAAlP3cpIKh4UKiSU9q9uBDJqE # ZkMv+9YBWNflvwnOGXL2AYJ0r+qLqL5zFnRLzHoHbKM9tl90FV8f80Gn/UufvFt4 # 4RMA6fs5P0PdQa3Sr4qJaBjjYecuPKGXVsC7kd+CvIA7cMJoh1Xa2O+QlrLao6cX # sOCPxrrQpBP1CB8l/BeevdkqJtgyNpRsI0gOfHPbKQIDAQABo4IBSTCCAUUwHQYD # VR0OBBYEFG/Oe4n1JTaDmX/n9v7kIGJtscXdMB8GA1UdIwQYMBaAFJ+nFV0AXmJd # g/Tl0mWnG1M1GelyMF8GA1UdHwRYMFYwVKBSoFCGTmh0dHA6Ly93d3cubWljcm9z # b2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0El # MjAyMDEwKDEpLmNybDBsBggrBgEFBQcBAQRgMF4wXAYIKwYBBQUHMAKGUGh0dHA6 # Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwVGlt # ZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3J0MAwGA1UdEwEB/wQCMAAwFgYDVR0l # AQH/BAwwCgYIKwYBBQUHAwgwDgYDVR0PAQH/BAQDAgeAMA0GCSqGSIb3DQEBCwUA # A4ICAQCJPg1S87aXD7R0My2wG7GZfajeVVCqdCS4hMAYgYKAj6yXtmpk5MN3wurG # wfgZYI9PO1ze2vwCAG+xgjNaMXCKgKMJA4OrWgExY3MSwrNyQfEDNijlLN7s4+Qw # DcMDFWrhJJHzL5NELYZw53QlF5nWU+WGU+X1cj7Pw6C04+ZCcsuI/2rOlMfAXN76 # xupKfxx6R24xl0vIcmTc2LDcCeCVT9ZPMaxAB1yH1JVXgseJ9SebBN/SLTuIq1OU # 2SrdvHWLJaDs3uMZkAFFZPaZf5gBUeUrbu32f5a1hufpw4k1fouwfzE9UFFgAhFW # RawzIQB2g/12p9pnPBcaaO5VD3fU2HMeOMb4R/DXXwNeOTdWrepQjWt7fjMwxNHN # lkTDzYW6kXe+Jc1HcNU6VL0kfjHl6Z8g1rW65JpzoXgJ4kIPUZqR9LsPlrI2xpnZ # 76wFSHrYpVOWESxBEdlHAJPFuLHVjiInD48M0tzQd/X2pfZeJfS7ZIz0JZNOOzP1 # K8KMgpLEJkUI2//OkoiWwfHuFA1AdIxsqHT/DCfzq6IgAsSNrNSzMTT5fqtw5sN9 # TiH87/S+ZsXcExH7jmsBkwARMmxEM/EckKj/lcaFZ2D8ugnldYGs4Mvjhg2s3sVG # ccQACvTqx+Wpnx55XcW4Mp0/mHX1ZScZbA7Uf9mNTM6hUaJXeDCCB3EwggVZoAMC # AQICEzMAAAAVxedrngKbSZkAAAAAABUwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNV # BAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4w # HAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29m # dCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAyMDEwMB4XDTIxMDkzMDE4MjIy # NVoXDTMwMDkzMDE4MzIyNVowfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp # bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw # b3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAw # ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDk4aZM57RyIQt5osvXJHm9 # DtWC0/3unAcH0qlsTnXIyjVX9gF/bErg4r25PhdgM/9cT8dm95VTcVrifkpa/rg2 # Z4VGIwy1jRPPdzLAEBjoYH1qUoNEt6aORmsHFPPFdvWGUNzBRMhxXFExN6AKOG6N # 7dcP2CZTfDlhAnrEqv1yaa8dq6z2Nr41JmTamDu6GnszrYBbfowQHJ1S/rboYiXc # ag/PXfT+jlPP1uyFVk3v3byNpOORj7I5LFGc6XBpDco2LXCOMcg1KL3jtIckw+DJ # j361VI/c+gVVmG1oO5pGve2krnopN6zL64NF50ZuyjLVwIYwXE8s4mKyzbnijYjk # lqwBSru+cakXW2dg3viSkR4dPf0gz3N9QZpGdc3EXzTdEonW/aUgfX782Z5F37Zy # L9t9X4C626p+Nuw2TPYrbqgSUei/BQOj0XOmTTd0lBw0gg/wEPK3Rxjtp+iZfD9M # 269ewvPV2HM9Q07BMzlMjgK8QmguEOqEUUbi0b1qGFphAXPKZ6Je1yh2AuIzGHLX # pyDwwvoSCtdjbwzJNmSLW6CmgyFdXzB0kZSU2LlQ+QuJYfM2BjUYhEfb3BvR/bLU # HMVr9lxSUV0S2yW6r1AFemzFER1y7435UsSFF5PAPBXbGjfHCBUYP3irRbb1Hode # 2o+eFnJpxq57t7c+auIurQIDAQABo4IB3TCCAdkwEgYJKwYBBAGCNxUBBAUCAwEA # ATAjBgkrBgEEAYI3FQIEFgQUKqdS/mTEmr6CkTxGNSnPEP8vBO4wHQYDVR0OBBYE # FJ+nFV0AXmJdg/Tl0mWnG1M1GelyMFwGA1UdIARVMFMwUQYMKwYBBAGCN0yDfQEB # MEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMv # RG9jcy9SZXBvc2l0b3J5Lmh0bTATBgNVHSUEDDAKBggrBgEFBQcDCDAZBgkrBgEE # AYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB # /zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvXzpoYxDBWBgNVHR8ETzBNMEug # SaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9N # aWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsG # AQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jv # b0NlckF1dF8yMDEwLTA2LTIzLmNydDANBgkqhkiG9w0BAQsFAAOCAgEAnVV9/Cqt # 4SwfZwExJFvhnnJL/Klv6lwUtj5OR2R4sQaTlz0xM7U518JxNj/aZGx80HU5bbsP # MeTCj/ts0aGUGCLu6WZnOlNN3Zi6th542DYunKmCVgADsAW+iehp4LoJ7nvfam++ # Kctu2D9IdQHZGN5tggz1bSNU5HhTdSRXud2f8449xvNo32X2pFaq95W2KFUn0CS9 # QKC/GbYSEhFdPSfgQJY4rPf5KYnDvBewVIVCs/wMnosZiefwC2qBwoEZQhlSdYo2 # wh3DYXMuLGt7bj8sCXgU6ZGyqVvfSaN0DLzskYDSPeZKPmY7T7uG+jIa2Zb0j/aR # AfbOxnT99kxybxCrdTDFNLB62FD+CljdQDzHVG2dY3RILLFORy3BFARxv2T5JL5z # bcqOCb2zAVdJVGTZc9d/HltEAY5aGZFrDZ+kKNxnGSgkujhLmm77IVRrakURR6nx # t67I6IleT53S0Ex2tVdUCbFpAUR+fKFhbHP+CrvsQWY9af3LwUFJfn6Tvsv4O+S3 # Fb+0zj6lMVGEvL8CwYKiexcdFYmNcP7ntdAoGokLjzbaukz5m/8K6TT4JDVnK+AN # uOaMmdbhIurwJ0I9JZTmdHRbatGePu1+oDEzfbzL6Xu/OHBE0ZDxyKs6ijoIYn/Z # cGNTTY3ugm2lBRDBcQZqELQdVTNYs6FwZvKhggNWMIICPgIBATCCAQGhgdmkgdYw # gdMxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsT # JE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJhdGlvbnMgTGltaXRlZDEnMCUGA1UECxMe # blNoaWVsZCBUU1MgRVNOOjQwMUEtMDVFMC1EOTQ3MSUwIwYDVQQDExxNaWNyb3Nv # ZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMKAQEwBwYFKw4DAhoDFQCEY0cP9rtDRtAt # ZUb0m4bGAtFex6CBgzCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNo # aW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29y # cG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEw # MA0GCSqGSIb3DQEBCwUAAgUA6/IxgDAiGA8yMDI1MDYxMDA0MjIyNFoYDzIwMjUw # NjExMDQyMjI0WjB0MDoGCisGAQQBhFkKBAExLDAqMAoCBQDr8jGAAgEAMAcCAQAC # AgONMAcCAQACAhLHMAoCBQDr84MAAgEAMDYGCisGAQQBhFkKBAIxKDAmMAwGCisG # AQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEAAgMBhqAwDQYJKoZIhvcNAQELBQAD # ggEBALIoJltkHEylK+ia72oiHlGrcBPw/bfPIe4fsDbmgNgrb2QRK6lEZI6dBOPn # 3exE3R64MW7mpSx8GD2bY+RPjx0D8/wFSLYdro7YJD8vzur3ptETC3ZGUTqOEUcc # TNc3bjabbCaqZ+hRWlif0se8zHusfhmGd/2wiSgaLv7smFpMJPyJ7k7KKxWw04mr # 9PsMBN1ATOvRs1U2n6P8hMnLeTcHnwURs+EerR5Aw88s3V5KsH5Fo5GjfXtpPh9w # Ue4Dr/GNKOusM/eBQGB3jgzvqCZWjwp5GxmcQPfGbMj/oErB+DE5rfyK5TgbUMtB # II/SCZq4TLqQjFWBTDKGr8tFaVUxggQNMIIECQIBATCBkzB8MQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGlt # ZS1TdGFtcCBQQ0EgMjAxMAITMwAAAf7QqMJ7NCELAQABAAAB/jANBglghkgBZQME # AgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJ # BDEiBCCOgn88Twns6IEmPaeQ0JIalSWTKTUYTC17IDXZBHrc5jCB+gYLKoZIhvcN # AQkQAi8xgeowgecwgeQwgb0EIBGFzN38U8ifGNH3abaE9apz68Y4bX78jRa2QKy3 # KHR5MIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAH+ # 0KjCezQhCwEAAQAAAf4wIgQg5+EHJF5XyjyP6gGfvDyMj7G/pFuzb4sgF03zWS/p # XnYwDQYJKoZIhvcNAQELBQAEggIAaKhocq5xVFLAU2mKnFMmR16bBAsOiDTUSKIO # vhRxWtyVKLICKsSUdFS6EBwb2G83afXDJU4dg+IEZvR8uGr67czLH3VLSrXUrNjv # Hd2RJ0J7OSWjuk3qqPpHx8iLgN6XK8ZoBrrGJkDmUz8u/DniNo4v/TOeC4AVKdNC # W+YPPzd2iU7dXHYanWrnHQKZ601AMV7QXHIsUDnfhWe727L+y+YM+nHEYei6aRzY # zsunp2tUTXhBR7UbKBGHcQkSOMGVSTIdt/S6lPEd1QBCfhEOIbZuxeKzSnqAXChw # KG3Bt1U5fHHd0D0ctCvzdPr5EV0/OiL5eVXux0bEKfzqePEkcWSr1+seF13YMbKK # kg55u4EzWTH/Q0GbN5tE/U8KJxxY6xL3bf9a3K/yK9pW3dyeVtDkEUlVgDco+lCJ # Mq0c8gNxoviuzweQ6X4FFvXOGvyql9OsEvff7uT7U5hkhjNKJphLTqYYWy3JKA+k # cQK78yJXaSBHwrv1E26wIcAXsaPx86JKp3IK/2AgkXTSIX+PlLUs9w93UeV1v7TM # 6oF3QVUrHD2dK2ofLNnfYf9RIWLMlo4n0OoS3sV6b9qiblEAcdk13U6GiwUGw6Ql # 8mG9oyhkPtCSLhY4kPXoExPGMXTDcKVJUkvlWI1hoLpHLveldhutiISIEaNWnErk # tMIVMEc= # SIG # End signature block |