Obs/bin/ObsDep/content/Powershell/ObservabilityConfig.psm1
|
<###################################################
# # # Copyright (c) Microsoft. All rights reserved. # # # ##################################################> class ObservabilityConfig { static RegisterObservabilityEventSource([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } $eventSources = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EventSources.EventSource $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] foreach ($eventSource in $eventSources) { $eventSourceRootName = $eventSource.EventSourceRootName $eventSourceLogName = $eventSource.EventSourceLogName $eventSourceLogSizeInBytes = $eventSource.EventSourceLogSizeInBytes $eventSourcePublisher = $eventSource.EventSourcePublisher Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog" try { # Register eventsource on hosts Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" $manifestDllPath = "$observabilityNugetPath\lib\net472\$Using:eventSourceRootName.dll" $manifestPath = "$observabilityNugetPath\lib\net472\$Using:eventSourceRootName.man" # Grant read only access to manifest files Trace-Execution "Granting read only access to $manifestPath" icacls $manifestPath /grant Everyone:R Trace-Execution "Granting read only access to $manifestDllPath" icacls $manifestDllPath /grant Everyone:R $publisherList = wevtutil ep if ($publisherList.Contains($Using:eventSourcePublisher)) { Trace-Execution "Publisher $Using:eventSourcePublisher already exists. Uninstalling." wevtutil uninstall-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath" Trace-Execution "Successfully uninstalled publisher $Using:eventSourcePublisher." } # Register the EventSource with Windows Trace-Execution "wevtutil installing manifest $manifestPath with resourceFilePath and messageFilePath $manifestDllPath" wevtutil install-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath" Trace-Execution "wevtutil setting log size to $Using:eventSourceLogSizeInBytes" wevtutil set-log $Using:eventSourceLogName /MaxSize:"$Using:eventSourceLogSizeInBytes" } } catch { Trace-Execution "Registering Observability EventSource failed with error: $_" throw $_ } Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog suceeded." } } static CreateObservabilityVolume([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $volumeLabel = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityVolumeLabel $volumeSize = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveSizeInBytes $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveAccessPath.Path $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath) $PhysicalDriveLetter = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.PhysicalDriveLetter $volumeFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveFileName $volumeLocation = "${PhysicalDriveLetter}:" $volumePath = Join-Path -Path $volumeLocation -ChildPath $volumeFileName Write-ObservabilityVolumeCreationStartTelemetry ` -ComputerNames ($hostIps -join " ") ` -VolumeFilePath $volumePath ` -VolumeAccessPath $driveAccessPath ` -VolumeLabel $volumeLabel ` -VolumeSize $volumeSize $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) # Add Observability Volume to each Host $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $VolumePath, [string] [parameter(Mandatory=$true)] $AccessPath, [string] [parameter(Mandatory=$true)] $VolumeLocation, [string] [Parameter(Mandatory=$true)] $VolumeLabel, [string] [Parameter(Mandatory=$true)] $VolumeSize ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" if (Test-Path -Path $AccessPath) { Write-ObservabilityLog "Folder $AccessPath already present. Skipping Observability volume creation." } else { Add-ObservabilityVolumeWithRetry -Path $VolumePath -AccessPath $AccessPath -VolumeLabel $VolumeLabel -Size $VolumeSize -StaticSize Add-VolumeAccessPath -AccessPath $AccessPath -VolumePath $VolumePath Add-MountVolumeScheduledTask -Path $VolumePath } } $argList = @($volumePath, $driveAccessPath, $volumeLocation, $volumeLabel, $volumeSize) Invoke-Command ` -ComputerName $hostIps ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $localAdminCredential ` -Authentication Credssp Write-ObservabilityLog "Observability Volume Setup Succeeded" } static SetUptimeScheduledTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityUptimeHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] Write-InfoLog "Obtaining local admin credential..." $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) Write-InfoLog "Starting Uptime scheduled task Setup" Write-InfoLog "Hosts: $hostNames" $scriptBlock = { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityUptimeHelpers.psm1" $hostName = $env:COMPUTERNAME Write-InfoLog "Starting uptime scheduled task on host $hostName." try { Set-UptimeTaskWithRetry Write-InfoLog "Uptime scheduled task on host $hostName succeeded." } catch { Write-ErrorLog "Uptime scheduled task on host $hostName failed with exception $_ ." throw $_ } } Invoke-Command -ComputerName $hostIps -ScriptBlock $scriptBlock -Credential $localAdminCredential -Authentication Credssp | Out-Null Write-InfoLog "Uptime scheduled task completed." } static SetObservabilityEventTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters, $eventName, $eventScriptBlock) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name Write-InfoLog "Starting $eventName events task Setup" Write-InfoLog "Hosts: $hostNames" $exceptionMessage = "" $scriptBlock = $eventScriptBlock $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] foreach ($hostIp in $hostIps) { Write-InfoLog "Starting $eventName events task on host $hostIp." $result = Invoke-Command -ComputerName $hostIp -ScriptBlock $scriptBlock -Credential $localAdminCredential -Authentication Credssp if($result -eq $true) { Write-InfoLog "$eventName events task on host $hostIp succeeded." } else { Write-ErrorLog "$eventName events task on host $hostIp failed with exception $result." $exceptionMessage += "${hostIp}: $result`n" } } if($exceptionMessage) { throw $exceptionMessage } } static SetCensusEventScheduledTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1" $scriptBlock = { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1" Set-CensusTaskWithRetry } [ObservabilityConfig]::SetObservabilityEventTask($Parameters, "CensusEvent", $scriptBlock) Write-InfoLog "Census events scheduled task completed." } static SetRegistrationEventOneTimeTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1" $scriptBlock = { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1" Set-RegistrationEventsTaskWithRetry } [ObservabilityConfig]::SetObservabilityEventTask($Parameters, "RegistrationEvents", $scriptBlock) Write-InfoLog "Registration events one-time task completed." } static CreateVolumeFoldersAndPruner([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $folderCleanupThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaCleanupThresholdInPercent $folderFreeSpaceThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaFreeSpaceThresholdInPercent $purgeFolderFrequencyInMinutes = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.PurgeFolderFrequencyInMinutes $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilitySubFolderConfigFileName $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveAccessPath.Path $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath) Write-ObservabilityLog "Starting Observability Volume Folder and Pruner Setup" Write-ObservabilityLog "Hosts: $hostNames" Write-ObservabilityLog "Volume folder mount access path: $driveAccessPath" Write-ObservabilityLog "Observability subfolder folder cleanup threshold: $folderCleanupThresholdPercent" Write-ObservabilityLog "Observability subfolder folder free space threshold: $folderFreeSpaceThresholdPercent" Write-ObservabilityLog "Observability volume purge frequency in minutes: $purgeFolderFrequencyInMinutes" $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) # Add Observability Volume to each Host $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $AccessPath, [string] [Parameter(Mandatory=$true)] $FolderCleanupThresholdPercent, [string] [Parameter(Mandatory=$true)] $FolderFreeSpaceThresholdPercent, [string] [Parameter(Mandatory=$true)] $PurgeFolderFrequencyInMinutes, [string] [Parameter(Mandatory=$true)] $SubFolderConfigFileName ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" New-VolumeFoldersAndPrunerWithRetry ` -AccessPath $AccessPath ` -CleanupThresholdPercent $FolderCleanupThresholdPercent ` -FreeSpaceThresholdPercent $FolderFreeSpaceThresholdPercent ` -PurgeFolderFrequencyInMinutes $PurgeFolderFrequencyInMinutes ` -SubFolderConfigFileName $SubFolderConfigFileName } $argList = @( $driveAccessPath, $folderCleanupThresholdPercent, $folderFreeSpaceThresholdPercent, $purgeFolderFrequencyInMinutes, $subFolderConfigFileName ) Invoke-Command ` -ComputerName $hostIps ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $localAdminCredential ` -Authentication Credssp Write-ObservabilityLog "Observability subfolder and pruner Setup Succeeded" } # Set registry key HKLM\Software\Microsoft\AzureStack DeviceType to AzureEdge static SetDeviceTypeRegistryKey([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "SetDeviceTypeRegistryKey start." $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) try { Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { Write-Verbose "Setting HKLM\Software\Microsoft\AzureStack DeviceType registry key to AzureEdge." $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\' if (!(Test-Path -Path $registryPath)) { New-Item -Path $registryPath -Force } New-ItemProperty -Path $registryPath -Name 'DeviceType' -PropertyType 'String' -Value "AzureEdge" -Force Write-Verbose "Finished DeviceType Registry key setup on $($env:COMPUTERNAME)" } Write-ObservabilityLog "Succeeded in setting AzureStack DeviceType Registry key." } catch { Write-ObservabilityErrorLog "[Error] DeviceType registry key set up failed with an error : $_ " throw $_ } } # Set up AzureStack environment to enable Autologger and UTC Telemetry via MA # Reg key set up needs to happend upfornt during host configuration static SetUpUtcExporterFeature([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] Write-ObservabilityLog "Set up AzureStack environment to enable Autologger and UTC Telemetry via MA." $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) try { Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $utcExporterPath = "$($env:SystemRoot)\System32\UtcExporters" # Create UtcExporters folder in system32 if (-not (Test-Path -Path $utcExporterPath)) { New-Item -Path "$($env:SystemRoot)\System32" -ItemType Directory -Name 'UtcExporters' -Force } $GenevaNameSpace = "AEOprdTely" if(Test-IsCIEnv) { $GenevaNameSpace = "AEOppeTely" } Write-ObservabilityLog "Set up the required regkey to enable utc exporter Feature" New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack' -Name 'AllowExporters' -PropertyType 'DWORD' -Value 1 -Force New-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters' -Force New-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Force New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Name 'DllPath' -PropertyType 'String' -Value "$($utcExporterPath)\UtcGenevaExporter.dll" -Force New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Name 'GenevaNamespace' -PropertyType 'String' -Value $GenevaNameSpace -Force New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\TestHooks' -Name 'SkipSignatureMitigation' -PropertyType 'DWORD' -Value 1 -Force # Update UTC Service binary Write-ObservabilityLog "Updating UTC service dll" Stop-Service "diagtrack" -Force Start-Sleep -Seconds 10 Write-ObservabilityLog "Copy UTC Exporter dll to $($utcExporterPath)" Copy-Item -Path "$observabilityNugetPath\lib\net472\UtcGenevaExporter.dll" -Destination $utcExporterPath -Force Start-Service "diagtrack" Write-ObservabilityLog "Finished UTC Exporter setup on $($env:COMPUTERNAME)" } Write-ObservabilityLog "Set up AzureStack environment to enable Autologger and UTC Telemetry via MA succeeded." } catch { Write-ObservabilityErrorLog "[Error] UTC Exporter set up failed with an error : $_ " throw $_ } } # Install VC Runtime on all hosts static InstallVcRuntime([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] Write-ObservabilityLog "Starting installation of X64 VC Redistributable." $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) try { Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "[Starting] installation of VC Redistributable on host $($env:COMPUTERNAME)" $vcRedistFilePath = "$observabilityNugetPath\content\VS17\VC_redist.x64.exe" if(Test-Path -Path $vcRedistFilePath) { Start-Process -File $vcRedistFilePath -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow } else { $errMsg = "[ERROR] VC Redistributable [$vcRedistilePath] not found on host $($env:COMPUTERNAME)" Write-ObservabilityErrorLog $errMsg throw $errMsg } Write-ObservabilityLog "[Finished] installation of VC Redistributable on host $($env:COMPUTERNAME)" } } catch { Write-ObservabilityErrorLog "[Failed] installation of VC Redistributable: $_ " throw $_ } Write-ObservabilityLog "Finished installation of X64 VC Redistributable." } static CreateObservabilityVolumeOnNC([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "Starting observability volume setup on NC VMs" $ncVMs = $Parameters.Roles["NC"].PublicConfiguration.Nodes.Node.Name $volumeFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveFileName $volumeLabel = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityVolumeLabel $volumeSize = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveSizeInBytes $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveAccessPath.Path $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath) $folderCleanupThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaCleanupThresholdInPercent $folderFreeSpaceThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaFreeSpaceThresholdInPercent $purgeFolderFrequencyInMinutes = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.PurgeFolderFrequencyInMinutes $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilitySubFolderConfigFileName $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $VolumePath, [string] [parameter(Mandatory=$true)] $HostComputerName, [string] [parameter(Mandatory=$true)] $AccessPath, [string] [parameter(Mandatory=$true)] $FolderCleanupThresholdPercent, [string] [parameter(Mandatory=$true)] $FolderFreeSpaceThresholdPercent, [string] [Parameter(Mandatory=$true)] $PurgeFolderFrequencyInMinutes, [string] [parameter(Mandatory=$true)] $SubFolderConfigFileName, [PSCredential] [parameter(Mandatory=$true)] $Credential ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $remoteVolumePath = "\\$HostComputerName\$VolumePath" $remoteVolumePath = $remoteVolumePath -replace ":", "$" Add-VolumeAccessPath -AccessPath $AccessPath -VolumePath $remoteVolumePath Add-MountVolumeScheduledTask -Path $remoteVolumePath -Credential $Credential New-VolumeFoldersAndPrunerWithRetry ` -AccessPath $AccessPath ` -CleanupThresholdPercent $FolderCleanupThresholdPercent ` -FreeSpaceThresholdPercent $FolderFreeSpaceThresholdPercent ` -PurgeFolderFrequencyInMinutes $PurgeFolderFrequencyInMinutes ` -SubFolderConfigFileName $SubFolderConfigFileName Set-FolderQuotas -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName } if(-not($ncVMs -is [System.Array])) # Only 1 VM exists { $ncVMs = @($ncVMs) } foreach($ncVM in $ncVMs) { $hostComputerName = $env:COMPUTERNAME Write-ObservabilityLog "Starting observability volume onboarding for $ncVM." try { Get-VM $ncVM | Out-Null # Test that VM is accessbile } catch { Write-ObservabilityLog "$ncVM not found on $hostComputerName. No op." continue } $vmHardDrivePath = (Get-VMHardDiskDrive $ncVM).Path if($vmHardDrivePath -is [System.Array]) { $vmHardDrivePath = $vmHardDrivePath[0] } $volumeLocation = Split-Path $vmHardDrivePath -Parent $volumePath = Join-Path -Path $volumeLocation -ChildPath $volumeFileName if(Test-Path $driveAccessPath) { Write-Observability "Folder $driveAccessPath already created. Skipping Observability Volume creation." } else { Write-ObservabilityLog "Adding observability volume at volume path $volumePath" Add-ObservabilityVolumeWithRetry -Path $volumePath -AccessPath $driveAccessPath -VolumeLabel $volumeLabel -Size $volumeSize -StaticSize if((Get-DiskImage $volumePath).Attached) { Dismount-VHD $volumePath } $argList = @( $volumePath, $hostComputerName, $driveAccessPath, $folderCleanupThresholdPercent, $folderFreeSpaceThresholdPercent, $purgeFolderFrequencyInMinutes, $subFolderConfigFileName, $domainUserCredential ) Invoke-Command ` -ComputerName $ncVM ` -ScriptBlock $scriptBlock ` -Credential $domainUserCredential ` -Authentication Credssp ` -ArgumentList $argList Write-ObservabilityLog "Finished observability volume onboarding for $ncVM." } } } static RegisterObservabilityEventSourceOnNC([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $eventSources = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EventSources.EventSource $ncVMs = $Parameters.Roles["NC"].PublicConfiguration.Nodes.Node.Name $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) foreach ($eventSource in $eventSources) { $eventSourceRootName = $eventSource.EventSourceRootName $eventSourceLogName = $eventSource.EventSourceLogName $eventSourceLogSizeInBytes = $eventSource.EventSourceLogSizeInBytes Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog" $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $EventSourceRootName, [string] [parameter(Mandatory=$true)] $EventSourceLogName, [string] [parameter(Mandatory=$true)] $EventSourceLogSizeInBytes ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" $manifestDllPath = "$observabilityNugetPath\lib\net472\$EventSourceRootName.dll" $manifestPath = "$observabilityNugetPath\lib\net472\$EventSourceRootName.man" # Grant read only access to manifest files Trace-Execution "Granting read only access to $manifestPath" icacls $manifestPath /grant Everyone:R Trace-Execution "Granting read only access to $manifestDllPath" icacls $manifestDllPath /grant Everyone:R # Register the EventSource with Windows Trace-Execution "wevtutil installing manifest $manifestPath with resourceFilePath and messageFilePath $manifestDllPath" wevtutil install-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath" Trace-Execution "wevtutil setting log size to $EventSourceLogSizeInBytes" wevtutil set-log $EventSourceLogName /MaxSize:"$EventSourceLogSizeInBytes" } try { # Register eventsource on hosts Invoke-Command ` -ComputerName $ncVMs ` -ScriptBlock $scriptBlock ` -Credential $domainUserCredential ` -Authentication Credssp ` -ArgumentList @($eventSourceRootName, $eventSourceLogName, $eventSourceLogSizeInBytes) } catch { Trace-Execution "Registering Observability EventSource failed with error: $_" throw $_ } Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog suceeded." } } static SetFolderQuotas ([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilitySubFolderConfigFileName $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveAccessPath.Path $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath) Write-ObservabilityLog "Starting observability volume fsrm quota setup on $hostNames" $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $AccessPath, [string] [parameter(Mandatory=$true)] $SubFolderConfigFileName ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Set-FolderQuotas -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName } try { Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock $scriptBlock ` -ArgumentList @($driveAccessPath, $subFolderConfigFileName) } catch { Write-ObservabilityErrorLog "[Failed] setup of FSRM Quotas: $_ " throw $_ } } static [PSCredential] GetLocalCredential([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "Obtaining local admin credential..." $localAdminCredential = $null try { $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $securityInfo = $cloudRole.PublicInfo.SecurityInfo $localAdmin = $securityInfo.LocalUsers.User | Where-Object Role -eq $Parameters.Configuration.Role.PrivateInfo.Accounts.BuiltInAdminAccountID $localAdminCredential = $Parameters.GetCredential($localAdmin.Credential) } catch { Write-ObservabilityErrorLog "Failed to obtain local admin credentials: $_ " throw $_ } Write-ObservabilityLog "Local admin credential obtained." if ($localAdminCredential.UserName -eq "removed") { Write-ObservabilityLog "The local admin credential obtained has been removed. Using domain credential instead." return [ObservabilityConfig]::GetDomainCredential($Parameters) } return $localAdminCredential } static [PSCredential] GetDomainCredential([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "Obtaining domain user credentials..." $domainUserCredential = $null try { $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $securityInfo = $cloudRole.PublicInfo.SecurityInfo $physicalMachinesRole = $Parameters.Configuration.Role $domainName = $Parameters.Roles.Domain.PublicConfiguration.PublicInfo.DomainConfiguration.Fqdn $domainUser = $securityInfo.DomainUsers.User | Where-Object Role -EQ $physicalMachinesRole.PrivateInfo.Accounts.DomainUserAccountID $domainUserCredential = $Parameters.GetCredential($domainUser.Credential) # Making sure that the credential is domain qualified, PowerShell Direct require credential to be domain qualified if($($domainUserCredential.GetNetworkCredential().Domain) -eq "") { $domainUserCredential = New-Object -TypeName PSCredential -ArgumentList "$domainName\$($domainUserCredential.UserName)", $($domainUserCredential.Password) } Write-ObservabilityLog "Domain user credentials obtained." } catch { Write-ObservabilityErrorLog "Failed to obtain domain user credentials: $_ " throw $_ } return $domainUserCredential } static InstallBootstrapObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $exceptionMessage = "" $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] try { Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability installation start." $deploymentLaunchType = Get-DeploymentLaunchype if ($deploymentLaunchType -eq "CloudDeployment") { Write-ObservabilityLog "$env:COMPUTERNAME This stamp was launched via CloudDeployment. Skipping Bootstrap Observability installation." return } $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $cloudId = $cloudRole.PublicInfo.CloudId $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare $proxyUrl = [ObservabilityConfig]::GetProxyUrl($Parameters) $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $AgentMsiPath, [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $TenantId, [string] [Parameter(Mandatory=$true)] $AccountId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $CloudId, [string] [Parameter(Mandatory=$true)] $EnvironmentName, [string] [Parameter(Mandatory=$true)] $Region, [string] [Parameter(Mandatory=$true)] $StampId ) $errorMessage = "" $setHostNICIPAddressLogFile = Join-Path -Path $env:systemdrive -ChildPath $LogFileRelativePath Start-Transcript -Append -Path $setHostNICIPAddressLogFile try { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $ResourceName = Get-ArcResourceName Write-ObservabilityLog "$env:COMPUTERNAME Going to install Arc-for-server agent..." Install-ArcForServerAgent ` -AgentMsiPath $AgentMsiPath ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -TenantId $TenantId ` -ResourceGroupName $ResourceGroupName ` -EnvironmentName $EnvironmentName ` -Region $Region ` -ResourceName $ResourceName ` -ProxyUrl $using:proxyUrl | Out-Null Write-ObservabilityLog "$env:COMPUTERNAME Going to download Observability Extension..." Install-ArcForServerExtensions ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -TenantId $TenantId ` -AccountId $AccountId ` -ResourceGroupName $ResourceGroupName ` -ResourceName $ResourceName ` -EnvironmentName $EnvironmentName ` -Region $Region | Out-Null $hostNameHash = Get-Sha256Hash -ClearString (hostname) $nodeId = "$($StampId)-$hostNameHash" Write-BootstrapNodeIdAndHardwareIdHashTelemetry -BootstrapNodeId $nodeId } catch{ $errorMessage = $PSItem.ToString() Write-ObservabilityErrorLog $errorMessage throw $PSItem.Exception.Message } finally { Write-ArcForServerInstallationStopTelemetry ` -ComputerName $env:COMPUTERNAME ` -Message "Bootstrap Observability installation end" ` -ExceptionDetails $errorMessage } } $argList = @( $arcForServerMsiFilePath, $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.TenantId, $registrationParams.AccountId, $registrationParams.ResourceGroupName, $cloudId, $registrationParams.EnvironmentName, $registrationParams.Region, $registrationParams.ResourceName ) Invoke-Command ` -ComputerName $hostIps ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $localAdminCredential ` -Authentication Credssp } catch { $errMsg = "Bootstrap Observability installation failed with $_" $exceptionMessage += $errMsg Write-ObservabilityErrorLog $errMsg throw } Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability installation end." } static InstallRemoteSupportArcExtension([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $exceptionMessage = "" $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] try { Write-ObservabilityLog "$env:COMPUTERNAME Remote Support Arc installation start." $deploymentLaunchType = Get-DeploymentLaunchype if ($deploymentLaunchType -eq "CloudDeployment") { Write-ObservabilityLog "$env:COMPUTERNAME This stamp was launched via CloudDeployment. Skipping RemoteSupport Arc installation." return } $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) $scriptBlock = { param ( [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $TenantId, [string] [Parameter(Mandatory=$true)] $AccountId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $EnvironmentName, [string] [Parameter(Mandatory=$true)] $Region, [string] [Parameter(Mandatory=$true)] $StampId ) $errorMessage = "" $setHostNICIPAddressLogFile = Join-Path -Path $env:systemdrive -ChildPath $LogFileRelativePath Start-Transcript -Append -Path $setHostNICIPAddressLogFile try { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityConstants.psm1" $ResourceName = Get-ArcResourceName Write-ObservabilityLog "$env:COMPUTERNAME Going to download Remote Support extension..." Install-ArcForServerExtensions ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -TenantId $TenantId ` -AccountId $AccountId ` -ResourceGroupName $ResourceGroupName ` -ResourceName $ResourceName ` -EnvironmentName $EnvironmentName ` -Region $Region ` -Type $ObservabilityConfigConstants.RemoteSupportExtensionType | Out-Null # Bring extension out of listener mode if it is StartArcRemoteSupportAgent } catch { $errorMessage = $PSItem.ToString() Write-ObservabilityErrorLog $errorMessage throw $PSItem.Exception.Message } finally { Write-ArcForServerInstallationStopTelemetry ` -ComputerName $env:COMPUTERNAME ` -Message "Remote Support Arc extension installation end" ` -ExceptionDetails $errorMessage } } $argList = @( $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.TenantId, $registrationParams.AccountId, $registrationParams.ResourceGroupName, $registrationParams.EnvironmentName, $registrationParams.Region, $registrationParams.ResourceName ) Invoke-Command ` -ComputerName $hostIps ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $localAdminCredential ` -Authentication Credssp } catch { $errMsg = "Remote Support Arc Extension installation failed with $_" $exceptionMessage += $errMsg Write-ObservabilityErrorLog $errMsg throw } Write-ObservabilityLog "$env:COMPUTERNAME Remote Support Arc Extension installation end." } static UninstallBootstrapObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability uninstallation start." $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare Write-ObservabilityLog "$env:COMPUTERNAME Arc For Server MSI Path: $arcForServerMsiFilePath" $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $AgentMsiPath, [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $AccountId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $CloudId ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" # Import MiscConstants to remove GMAScenario reg key $gmaNugetName = "Microsoft.AzureStack.Observability.GenevaMonitoringAgent" $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $gmaNugetName) -ChildPath "content" Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking $gmaScenarioRegKeyPath = $MiscConstants.GMAScenarioRegKey.Path $gmaScenarioRegKeyName = $MiscConstants.GMAScenarioRegKey.Name if (Test-RegKeyExists -Path $gmaScenarioRegKeyPath -Name $gmaScenarioRegKeyName) { Remove-ItemProperty ` -Path $gmaScenarioRegKeyPath ` -Name $gmaScenarioRegKeyName ` -Force Write-ObservabilityLog "Removed registry key $gmaScenarioRegKeyName at path $gmaScenarioRegKeyPath" } Uninstall-ArcForServerExtensions ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -AccountId $AccountId ` -ResourceGroupName $ResourceGroupName ` -CloudId $CloudId $result = Uninstall-ArcForServerAgent ` -AgentMsiPath $AgentMsiPath ` -AccessToken $AccessToken return $result } # Install Arc for server agent on each Host $exceptionMessage = "" foreach($hostIp in $hostIps) { $argList = @($arcForServerMsiFilePath, $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.AccountId, $registrationParams.ResourceGroupName, $registrationParams.CloudId) $result = Invoke-Command ` -ComputerName $hostIp ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $domainUserCredential ` -Authentication Credssp if ($result -eq $true) { Write-ObservabilityLog "Arc for server agent setup on host $hostIp succeeded." } else { Write-ObservabilityLog "Arc for server agent setup on host $hostIp failed with exception $result." $exceptionMessage += "${hostIp}: $result`n" } } if($exceptionMessage) { throw $exceptionMessage } Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability uninstallation end." } static SetupArcAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" try { Write-ObservabilityLog "$env:COMPUTERNAME Arc Agent update start." $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare $errorMessage = "" Write-ObservabilityLog "$env:COMPUTERNAME Going to update Arc-for-server agent..." Update-ArcForServerAgent -AgentMsiPath $arcForServerMsiFilePath | Out-Null } catch{ $errorMessage = $PSItem.ToString() Write-ObservabilityErrorLog $errorMessage throw $PSItem.Exception.Message } Write-ObservabilityLog "$env:COMPUTERNAME Arc Agent update end." } static ConnectArcAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $exceptionMessage = "" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } try { Write-ObservabilityLog "$env:COMPUTERNAME Arc Agent connect start." $proxyUrl = [ObservabilityConfig]::GetProxyUrl($Parameters) $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) $scriptBlock = { param ( [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $TenantId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $EnvironmentName, [string] [Parameter(Mandatory=$true)] $Region, [string] [Parameter(Mandatory=$true)] $StampId ) $errorMessage = "" try { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $ResourceName = Get-ArcResourceName Write-ObservabilityLog "$env:COMPUTERNAME Going to connect Arc-for-server agent..." Connect-ArcForServerAgent ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -TenantId $TenantId ` -ResourceGroupName $ResourceGroupName ` -EnvironmentName $EnvironmentName ` -Region $Region ` -ResourceName $ResourceName ` -ProxyUrl $using:proxyUrl | Out-Null } catch{ $errorMessage = $PSItem.ToString() Write-ObservabilityErrorLog $errorMessage throw $PSItem.Exception.Message } } $argList = @( $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.TenantId, $registrationParams.ResourceGroupName, $registrationParams.EnvironmentName, $registrationParams.Region, $registrationParams.ResourceName ) $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) Invoke-Command ` -Credential $localAdminCredential ` -Authentication Credssp ` -ComputerName $hostNames ` -ScriptBlock $scriptBlock ` -ArgumentList $argList } catch { $exceptionMessage += "ArcAgent connect failed with $_" Write-ObservabilityErrorLog $exceptionMessage throw } Write-ObservabilityLog "$env:COMPUTERNAME Arc Agent connect end." } <# .SYNOPSIS Evaluation function to identify what type of GMA configuration action to use. Following are the cases involved during evaluation : case 1 : If the interface is invoked in case of Deployment then run configuration on all nodes. case 2 : If the interface is invoked in case of AddNode or RepairNode then run configuration on only that node. .EXAMPLE [GMATenantJson]::EvaluateGMAConfigurationType($Parameters) .PARAMETER Parameters The ECE role parameters for this interface. #> static [CloudEngine.Actions.ConditionalActionDescription] EvaluateListenerModeConfigurationType([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $ErrorActionPreference = "Stop" ## For classes in PS, we need to use the call stack to get the current executing function. $functionName = $(Get-PSCallStack)[0].FunctionName try { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $cloudRolePath = "ObservabilityConfig" $listenerModeOnAllNodes = "SetListenerModeOnAllNodes" $listenerModeOnOneNode = "SetListenerModeOnOneNode" $onAllNodesAction = [CloudEngine.Actions.ConditionalActionDescription]::CreateWithDefinedAction($cloudRolePath, $listenerModeOnAllNodes) $onOneNodeAction = [CloudEngine.Actions.ConditionalActionDescription]::CreateWithDefinedAction($cloudRolePath, $listenerModeOnOneNode) $nodeName = Get-ExecutionContextNodeName -Parameters $Parameters if($null -ne $nodeName) { # In add node or repair node case, ExecutionContextNodeName should have exactly one value. Trace-Execution "Execution context node was determined. Listener Mode will be set on $nodeName only." return $onOneNodeAction } # In regular deployment case, ExecutionContextNodeName will be null. Trace-Execution "No exceution context node was determined. Listener Mode will set on all the nodes." return $onAllNodesAction } catch { Trace-Error "$functionName :Failed to evaluate ListenerMode configuration type on node $($env:COMPUTERNAME) due to following error : $_" throw } } static StopArcExtensionObservabilityAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $hostName = Get-ExecutionContextNodeName -Parameters $Parameters # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIp = $allHosts[$hostName] Invoke-Command ` -ComputerName $hostIp ` -Credential $domainUserCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $ObsAgentServiceName = "AzureStack Observability Agent" $ObsAgentServiceDisplayName = "AzureStack Arc Extension Observability Agent" $stopDelayInSeconds = 10 $service = Get-Service | Where-Object {$_.Name -eq $ObsAgentServiceName -and $_.DisplayName -eq $ObsAgentServiceDisplayName } if($service) { if ($service.Status -ne "Stopped") { Write-ObservabilityLog "Stopping service $ObsAgentServiceName with display name $ObsAgentServiceDisplayName on host $($env:COMPUTERNAME)" $service | Stop-Service Write-ObservabilityLog "Sleeping $stopDelayInSeconds seconds after stopping $ObsAgentServiceName before unregistering." Start-Sleep -Seconds $stopDelayInSeconds } Write-ObservabilityLog "Unregistering service $ObsAgentServiceName." sc.exe delete $ObsAgentServiceName Write-ObservabilityLog "Successfully unregistered service $ObsAgentServiceDisplayName" } else { Write-ObservabilityLog "Service $ObsAgentServiceName not found. Noop." } } } static StopRemSupAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { # Making this no-op since it's going in the same payload as InstallRemoteSupportArcExtension # Will remove the action plan for this in another payload } static SetWatchdogToListenerMode([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $hostName = Get-ExecutionContextNodeName -Parameters $Parameters # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIp = $allHosts[$hostName] Invoke-Command ` -ComputerName $hostIp ` -Credential $domainUserCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\Observability' Write-ObservabilityLog "Setting $registryPath WatchdogListenerMode registry key to 1 on $($env:COMPUTERNAME)." if (!(Test-Path -Path $registryPath)) { New-Item -Path $registryPath -Force } New-ItemProperty -Path $registryPath -Name 'WatchdogListenerMode' -PropertyType 'DWord' -Value 1 -Force Write-ObservabilityLog "Finished setting WatchdogListenerMode Registry key to DWord 1 on $($env:COMPUTERNAME)" $timeLimitInSeconds = 600 $retryPeriod = 10 $elapsed = 0 $maHostProcess = Get-Process "MonAgentHost" -ErrorAction SilentlyContinue $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue while($maHostProcess -or $maCoreProcess) { if($maHostProcess) { $pids = $maHostProcess.Id | Out-String Write-ObservabilityLog "MonAgentHost Active Process Ids: `n $pids" } if($maCoreProcess) { $pids = $maCoreProcess.Id | Out-String Write-ObservabilityLog "MonAgentCore Active Process Ids: `n $pids" } if($elapsed -gt $timeLimitInSeconds) { $errMsg = "WatchdogListenerMode has failed to stop Monitoring agent processes after $timeLimitInSeconds." Write-ObservabilityErrorLog $errMsg throw $errMsg } Write-ObservabilityLog "Monitoring Agent processes have not yet stopped. Retrying in $retryPeriod seconds." Start-Sleep -Seconds $retryPeriod $elapsed += $retryPeriod $maHostProcess = Get-Process "MonAgentHost" -ErrorAction SilentlyContinue $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue } Write-ObservabilityLog "MonitoringAgent has been successfully stopped after Watchdog has been set to listener mode." } } static SetGMATenantJsonRegistryKeys([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "SetGMATenantJsonRegistryKeys start" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $gmaNugetName = "Microsoft.AzureStack.Observability.GenevaMonitoringAgent" $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $gmaNugetName) -ChildPath "content" Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking ## GMA cache folder path $gmaCacheFolderName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.GMACacheFolderName $gmaCacheFolderPath = Join-Path -Path $env:SystemDrive -ChildPath $gmaCacheFolderName ## Determine GcsEnvironment $gcsEnvironment = "Prod" ## default environment ## Check if the reg key created for CI exists or not, if yes then change the GCSEnvironment to point to PPE. if(Test-IsCIEnv) { $gcsEnvironment = "Ppe" } $envInfoFilePath = "$GmaPackageContentPath\EnvironmentInfo.json" $tenantInfoContent = Get-Content $envInfoFilePath -Raw | ConvertFrom-Json $envInfo = $tenantInfoContent.$GcsEnvironment # Settings from EnvironmentInfo.json $gcsEndpoint = $envInfo.EndPoint $gcsAccount = $envInfo.Account $genevaConfigVersion = $envInfo.ConfigVersion # Settings from ECE $assemblyVersion = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.Version $registrationSubscriptionId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationSubscriptionId $registrationResourceGroupName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationResourceGroupName $stampId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationResourceName $gcsRegionName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationRegion $clusterName = $Parameters.Roles["Cluster"].PublicConfiguration.Clusters.Node.Name $cloudId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.CloudId $deviceArmResourceUri = "/Subscriptions/$registrationSubscriptionId/resourceGroups/$registrationResourceGroupName/providers/Microsoft.AzureStackHCI/clusters/$stampId" $osBuild = Get-OSBuildVersion $registrationArcResourceGroupName = $registrationResourceGroupName $arcResourceGroupName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationArcServerResourceGroupName if (![String]::IsNullOrEmpty($arcResourceGroupName)) { Trace-Execution "Using RegistrationArcServerResourceGroupName $arcResourceGroupName for ArcServer resource in Tenant Json" $registrationArcResourceGroupName = $arcResourceGroupName } $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $Using:gmaNugetName) -ChildPath "content" Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking # Will unset GMA scenario registry path at bootstrap uninstall Set-GMAScenarioRegistryKeyToBootstrap | Out-Null # For Bootstrap Scenario, nodeid is stampid-sha256(hostname) $hostNameHash = Get-Sha256Hash -ClearString (hostname) $nodeId = "$($Using:stampId)-$hostNameHash" $arcAgentResourceId = Get-ArcResourceId -SubscriptionId $Using:registrationSubscriptionId -ResourceGroupName $Using:registrationArcResourceGroupName -CloudId $Using:cloudId $configTypes = @("Telemetry", "Diagnostics", "Health", "Metrics", "Security") foreach($configType in $configTypes) { $envInfo = $Using:envInfo $gcsNameSpace = $envInfo.Namespaces.$configType if ($configType -eq "Metrics") { $gcsNameSpace = Get-MetricsNamespaceRegionMapping -Region $using:gcsRegionName -MetricsNameSpace $envInfo.Namespaces.$configType $envInfo #Set environment variables required for 3P metrics Set-EnvironmentVariablesForMetrics ` -GcsEnvironment $using:gcsEnvironment ` -EnvInfoFilePath $using:envInfoFilePath ` -AssemblyBuildVersion $Using:assemblyVersion ` -ClusterName $Using:clusterName ` -HciResourceUri $Using:deviceArmResourceUri } # cacheLocalPath will be overwritten by Extension Install script because cache location for bootstrap is not known until install time $cacheLocalPath = Join-Path -Path $Using:gmaCacheFolderPath -ChildPath $($configType + "Cache") Set-TenantConfigRegistryKeys ` -ConfigType $configType ` -Version "1.0" ` -GcsAuthIdType "AuthMSIToken" ` -GcsEnvironment $Using:gcsEndpoint ` -GcsGenevaAccount $Using:gcsAccount ` -GcsNamespace $gcsNameSpace ` -GcsRegion $Using:gcsRegionName ` -GenevaConfigVersion $Using:genevaConfigVersion ` -LocalPath $cacheLocalPath ` -DisableUpdate "true" ` -DisableCustomImds "true" ` -MONITORING_AEO_REGION $Using:gcsRegionName ` -MONITORING_AEO_DEVICE_ARM_RESOURCE_URI $Using:deviceArmResourceUri ` -MONITORING_AEO_STAMPID $Using:stampId ` -MONITORING_AEO_CLUSTER_NAME $Using:clusterName ` -MONITORING_AEO_OSBUILD $Using:osBuild ` -MONITORING_AEO_ASSEMBLYBUILD $Using:assemblyVersion ` -MONITORING_AEO_NODEID $nodeId ` -MONITORING_AEO_NODE_ARC_RESOURCE_URI $arcAgentResourceId ` -MONITORING_AEO_CLUSTER_NODE_NAME "%COMPUTERNAME%" } $hostname = hostname Write-ObservabilityLog "SetGMATenantJsonRegistryKeys on $hostname succeeded." } } static GenerateTelemetryFromCachedFiles([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "$env:COMPUTERNAME Generate Telemetry from Cached files start." $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $domainCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $cloudId = $cloudRole.PublicInfo.CloudId # Get host names using execution context $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $CorrelationId ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" $logOrchestratorNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.LogOrchestrator" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $result = Invoke-CachedTelemetryFilesParsing ` -LogOrchestratorNugetPath $logOrchestratorNugetPath ` -CorrelationId $CorrelationId return $result } # Install Arc for server agent on each Host $exceptionMessage = "" foreach($hostName in $hostNames) { $argList = @($cloudId) $result = Invoke-Command ` -ComputerName $hostName ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $domainCredential ` -Authentication Credssp if ($result -eq $true) { Write-ObservabilityLog "Generate Telemetry from Cached files on host $hostName succeeded." } else { Write-ObservabilityLog "Generate Telemetry from Cached files on host $hostName failed with exception $result." $exceptionMessage += "${hostName}: $result`n" } } if($exceptionMessage) { throw $exceptionMessage } Write-ObservabilityLog "$env:COMPUTERNAME Generate Telemetry from Cached files end." } static SyncDiagnosticLevel([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Trace-Execution "Getting parameters for Set-AzStackHCI" -Verbose $computerName = Get-Cluster | Select-Object -expand Name $resourceId = Get-AzureStackHCI | Select-Object -expand AzureResourceUri $streamingData = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.StreamingDataClient Trace-Execution "Streaming Data = $streamingData" -Verbose $episodicDataUpload = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EpisodicDataUpload Trace-Execution "Episodic Data Upload = $episodicDataUpload" -Verbose if ($streamingData) { if ($episodicDataUpload) { $diagnosticLevel = "Enhanced" } else { $diagnosticLevel = "Basic" } } else { $diagnosticLevel = "Off" } Trace-Execution "Diagnostic Level will be set to $diagnosticLevel" -Verbose Trace-Execution "Calling GetRegistrationParameters" -Verbose $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) Trace-Execution "Calling Set-AzStackHCI. Parameters = ComputerName $computerName, ResourceId $resourceId, DiagnosticLevel $diagnosticLevel, AccountId $($registrationParams.AccountId)" -Verbose Set-AzStackHCI -ComputerName $computerName ` -ResourceId $resourceId ` -DiagnosticLevel $diagnosticLevel ` -ArmAccessToken $registrationParams.ArmAccessToken ` -AccountId $registrationParams.AccountId ` -Confirm:$false ` -Verbose $retryAttempt = 0 $retrySleepTimeInSeconds = 10 $Retries = 5 $success = $false while(-not($success) -and ($retryAttempt -lt $Retries)) { $retryAttempt = $retryAttempt + 1 Trace-Execution "Attempt $retryAttempt of $Retries" -Verbose try { Trace-Execution "Calling Sync-AzureStackHCI" -Verbose Sync-AzureStackHCI -Verbose Trace-Execution "Calling Get-AzureStackHCI" -Verbose $portalDiagnosticLevel = Get-AzureStackHCI | Select-Object -expand DiagnosticLevel if ($portalDiagnosticLevel -eq $diagnosticLevel) { Trace-Execution "Portal diagnostic level was successfully set to $portalDiagnosticLevel" -Verbose $success = $true } else { throw "The Diagnostic Level does not match. Portal was not set to $diagnosticLevel, instead is $portalDiagnosticLevel" } } catch { if ($retryAttempt -lt $Retries) { $exceptionMessage = $_.Exception.Message Trace-Execution "Failure during syncing diagnostic level: '$exceptionMessage'. Retrying." -Verbose } else { Trace-Error "Syncing Diagnostic Level failed with error: $_" } Start-Sleep -Seconds $retrySleepTimeInSeconds } } Trace-Execution "Syncing Diagnostic Level successful. Set to: $diagnosticLevel" -Verbose } static UpdateGMATenantJsonNodeId([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $hostName = Get-ExecutionContextNodeName -Parameters $Parameters # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIp = $allHosts[$hostName] $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $assemblyBuild = "" try { Write-ObservabilityLog "Getting Assembly Build using GetStampVersion" Import-Module EceClient $ececlient = Create-ECEClusterServiceClient $assemblyBuild = $ececlient.GetStampVersion().GetAwaiter().GetResult() if ($assemblyBuild -eq "99.9999.9.10") { Write-ObservabilityLog "Getting Assembly Build using GetStampVersion returned $assemblyBuild, which is a possibly erroneous override. Using Assembly version in ECE parameters." $assemblyBuild = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.Version } } catch { Write-ObservabilityLog "Getting Assembly Build using GetStampVersion failed. Using Assembly version in ECE parameters" $assemblyBuild = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.Version } Write-ObservabilityLog "Assembly Build value of $assemblyBuild obtained." $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) Invoke-Command ` -Credential $localAdminCredential ` -Authentication Credssp ` -ComputerName $hostIp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $success = $false $extError = "" # Get NodeId $azureStackHciDetails = Get-AzureStackHCI $StampId = $azureStackHciDetails.AzureResourceName $nodeId = $StampId + "-" + (Get-ClusterNode(hostname)).Id $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\Observability\TenantJson' $nodeIdKey = "MONITORING_AEO_NODEID" $assemblyBuildKey = "MONITORING_AEO_ASSEMBLYBUILD" Write-ObservabilityLog "$($env:COMPUTERNAME): Setting keys $nodeIdKey to $nodeId and $assemblyBuildKey to $using:assemblyBuild at registry path $registryPath." if (!(Test-Path -Path $registryPath)) { New-Item -Path $registryPath -Force } New-ItemProperty -Path $registryPath -Name $nodeIdKey -PropertyType 'String' -Value $nodeId -Force New-ItemProperty -Path $registryPath -Name $assemblyBuildKey -PropertyType 'String' -Value $using:assemblyBuild -Force Write-ObservabilityLog "Finished setting registry keys $nodeIdKey and $assemblyBuildKey on $($env:COMPUTERNAME)" $paths = (Get-ChildItem "C:\Packages\Plugins\Microsoft.AzureStack.Observability*TelemetryAndDiagnostics*\*\" -ErrorAction SilentlyContinue).FullName foreach ($extRootPath in $paths) { Write-ObservabilityLog "Found Observability TelemetryAndDiagnostics extension at $extRootPath." if($success) { Write-ObservabilityLog "NodeId already updated successfully. Ignoring extension at $extRootPath." } else { try { Import-Module (Join-Path -Path $extRootPath -ChildPath 'scripts\ExtensionHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $logFile = Get-HandlerLogFile $gmaPackageContentPath = Get-GmaPackageContentPath Import-Module (Join-Path -Path "$gmaPackageContentPath" -ChildPath 'GMATenantJsonHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $cacheDirectories = New-CacheDirectories -LogFile $logFile $gmaCacheFolderPath = $cacheDirectories.GMACache $jsonDropLocation = Join-Path -Path $gmaCacheFolderPath -ChildPath "JsonDropLocation" foreach ($configType in $global:MiscConstants.ConfigTypes.Values) { $jsonConfigFileName = "AEO" + $configType + ".json" $tenantJsonFilePath = Join-Path -Path $jsonDropLocation -ChildPath $jsonConfigFileName Write-ObservabilityLog "Creating file $tenantJsonFilePath." Set-TenantConfigJsonFile -ConfigType $configType -FilePath $tenantJsonFilePath -LogFile $logFile Write-ObservabilityLog "Creation of file $tenantJsonFilePath succeeded." } $success = $true } catch { Write-ObservabilityErrorLog "Updating NodeId in tenant jsons failed with error $_" $extError = $_ } } } if (-not $success) { if ($extError) { throw "Updating NodeId in tenant jsons failed with error $extError" } else { $errMsg = "Could not update NodeId in tenant jsons because no TelemetryAndDiagnostics Extension was found at C:\Packages\Plugins." Write-ObservabilityErrorLog $errMsg throw $errMsg } } } } static DeleteObservabilityAgents([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $obsServices = @( "AzureStack Observability Agent", "AzureStack Observability FDA", "AzureStack Observability GenevaMonitoringAgent", "AzureStack Observability RemoteSupportAgent" ) $stopDelayInSeconds = 10 $agentManifestPath = "C:\Agents\AgentManifests" $obsServices | ForEach-Object { $service = Get-Service -Name $_ -ErrorAction SilentlyContinue | Where-Object {$_.DisplayName -notlike "*Arc Extension*"} if ($service) { try { Write-ObservabilityLog "Stopping service $($service.DisplayName)." $service | Stop-Service Write-ObservabilityLog "Successfully stopped service $($service.DisplayName). Sleeping for $stopDelayInSeconds before unregistering service." Start-Sleep -Seconds $stopDelayInSeconds Write-ObservabilityLog "Unregistering service $($service.DisplayName)." sc.exe delete $service.Name Write-ObservabilityLog "Stopping and unregistering of service $($service.DisplayName) succeeded." $manifestPath = "$agentManifestPath\$($service.DisplayName).json" if (Test-Path -Path $manifestPath) { Write-ObservabilityLog "Deleting agent manifest at $manifestPath." Remove-Item $manifestPath -Force Write-ObservabilityLog "Deletion of agent manifest at $manifestPath succeeded." } } catch { Write-ObservabilityErrorLog "Stopping and unregistering of service $($service.DisplayName) failed with error $_" } } } } static SetListenerModeOff([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\Observability' Write-ObservabilityLog "Setting $registryPath WatchdogListenerMode registry key to 0 on $($env:COMPUTERNAME)." if (!(Test-Path -Path $registryPath)) { New-Item -Path $registryPath -Force } New-ItemProperty -Path $registryPath -Name 'WatchdogListenerMode' -PropertyType 'DWord' -Value 0 -Force Write-ObservabilityLog "Finished setting WatchdogListenerMode Registry key to DWord 0 on $($env:COMPUTERNAME)" $timeLimitInSeconds = 600 $retryPeriod = 10 $elapsed = 0 $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue while($null -eq $maCoreProcess) { if($elapsed -gt $timeLimitInSeconds) { $errMsg = "Watchdog has failed to start Monitoring agent processes after $timeLimitInSeconds." Write-ObservabilityErrorLog $errMsg throw $errMsg } Write-ObservabilityLog "Monitoring Agent processes have not yet started. Retrying in $retryPeriod seconds." Start-Sleep -Seconds $retryPeriod $elapsed += $retryPeriod $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue } Write-ObservabilityLog "MonitoringAgent has been successfully started. Watchdog listener mode has been set to off." } static StartArcExtensionObsAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $success = $false $paths = (Get-ChildItem "C:\Packages\Plugins\Microsoft.AzureStack.Observability*TelemetryAndDiagnostics*\*\" -ErrorAction SilentlyContinue).FullName $extError = "" foreach ($extRootPath in $paths) { Write-ObservabilityLog "Found Observability TelemetryAndDiagnostics extension at $extRootPath." if ($success) { Write-ObservabilityLog "TelemetryAndDiagnostics Arc Extension Observability Agent already started. Ignoring extension at $extRootPath." } else { try { Import-Module (Join-Path -Path $extRootPath -ChildPath 'scripts\ExtensionHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $logFile = Get-HandlerLogFile ## Misc constants is defined here: https://msazure.visualstudio.com/One/_git/ASZ-Observability-MonitoringAgent?path=/src/GenevaMonitoringAgent/DeploymentScripts/GMATenantJsonHelper.psm1 ## Register Obs Agent as Windows service $obsAgent = $global:MiscConstants.ObsServiceDetails.ObsAgent $binaryFilePath = Join-Path -Path $global:ObsArtifactsPaths.ObservabilityAgent ` -ChildPath $obsAgent.BinaryFileName Write-ObservabilityLog "Registering $($obsAgent.DisplayName) service at path $binaryFilePath" Register-ServiceForObservability ` -ServiceName $obsAgent.Name ` -ServiceDisplayName $obsAgent.DisplayName ` -ServiceBinaryFilePath $binaryFilePath ` -LogFile $logFile ## Start Observability Agent. Write-ObservabilityLog "Starting $($obsAgent.DisplayName) service" Start-ServiceForObservability ` -ServiceName $obsAgent.Name ` -LogFile $logFile Write-ObservabilityLog "Observability Agent $($obsAgent.DisplayName) successfully registered and started" $success = $true } catch { Write-ObservabilityErrorLog "Starting TelemetryAndDiagnostics Arc Extension Observability Agent failed with error $_" $extError = $_ } } } if (-not $success) { if ($extError) { throw "Starting TelemetryAndDiagnostics Arc Extension Observability Agent failed with error $extError" } else { $errMsg = "Could not start Arc Extension Observability Agent because no Arc Extension was found at C:\Packages\Plugins." Write-ObservabilityErrorLog $errMsg throw $errMsg } } } static RegenerateGMATenantJsonFiles([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $success = $false $paths = (Get-ChildItem "C:\Packages\Plugins\Microsoft.AzureStack.Observability*TelemetryAndDiagnostics*\*\" -ErrorAction SilentlyContinue).FullName $extError = "" foreach ($extRootPath in $paths) { Write-ObservabilityLog "Found Observability TelemetryAndDiagnostics extension at $extRootPath." if($success) { Write-ObservabilityLog "GMA Tenant Json files already updated successfully. Ignoring extension at $extRootPath." } else { try { Import-Module (Join-Path -Path $extRootPath -ChildPath 'scripts\ExtensionHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $logFile = Get-HandlerLogFile $gmaPackageContentPath = Get-GmaPackageContentPath Import-Module (Join-Path -Path "$gmaPackageContentPath" -ChildPath 'GMATenantJsonHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $cacheDirectories = New-CacheDirectories -LogFile $logFile $gmaCacheFolderPath = $cacheDirectories.GMACache $jsonDropLocation = Join-Path -Path $gmaCacheFolderPath -ChildPath "JsonDropLocation" foreach ($configType in $global:MiscConstants.ConfigTypes.Values) { $jsonConfigFileName = "AEO" + $configType + ".json" $tenantJsonFilePath = Join-Path -Path $jsonDropLocation -ChildPath $jsonConfigFileName Write-ObservabilityLog "Creating file $tenantJsonFilePath." Set-TenantConfigJsonFile -ConfigType $configType -FilePath $tenantJsonFilePath -LogFile $logFile Write-ObservabilityLog "Creation of file $tenantJsonFilePath succeeded." } $success = $true } catch { Write-ObservabilityErrorLog "Regenerating tenant jsons failed with error $_" $extError = $_ } } } if (-not $success) { if ($extError) { throw "Regenerating GMA tenant jsons failed with error $extError" } else { $errMsg = "Could not regenerate GMA tenant jsons because no TelemetryAndDiagnostics Extension was found at C:\Packages\Plugins." Write-ObservabilityErrorLog $errMsg throw $errMsg } } } static RestoreMAWatchdogRegKeyAndService([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $functionName = "RestoreMAWatchdogRegKeyAndService" Write-ObservabilityLog "Starting $functionName." $HCITelemetryRegKey = @{ Path = 'HKLM:\SYSTEM\Software\Microsoft\MAWatchdogService\HCITelemetry' Name = 'AllowTelemetry' PropertyType = 'String' Value = 'True' } $regKey = $(Get-ItemProperty -Path $HCITelemetryRegKey.Path -Name $HCITelemetryRegKey.Name -ErrorAction SilentlyContinue) if($null -ne $regKey) { Write-ObservabilityLog "$functionName : Registry key $($HCITelemetryRegKey.Name) at $($HCITelemetryRegKey.Path) already exists. Noop." } else { Write-ObservabilityLog "$functionName : Registry key $($HCITelemetryRegKey.Name) at $($HCITelemetryRegKey.Path) does not exist. Restoring." $watchdogServiceName = "WatchdogAgent" if (-not (Test-Path -Path $HCITelemetryRegKey.Path)) { New-Item -Path $HCITelemetryRegKey.Path -Force } New-ItemProperty ` -Path $HCITelemetryRegKey.Path ` -Name $HCITelemetryRegKey.Name ` -PropertyType $HCITelemetryRegKey.PropertyType ` -Value $HCITelemetryRegKey.Value ` -Force | Out-Null Write-ObservabilityLog "$functionName : Created registry key $($HCITelemetryRegKey.Name) at $($HCITelemetryRegKey.Path) with value $($HCITelemetryRegKey.Value)." $watchdogAgent = Get-Service $watchdogServiceName -ErrorAction SilentlyContinue if($watchdogAgent) { Write-ObservabilityLog "$functionName : Restarting $watchdogServiceName windows service." $watchdogAgent | Restart-Service -ErrorAction SilentlyContinue } } } static [Hashtable] GetRegistrationParameters([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $cloudDeploymentNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.CloudDeployment" Import-Module Az.Accounts -Force $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } Trace-Execution "Getting registration parameters from ECE role parameters" $nodeNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $securityInfo = $cloudRole.PublicInfo.SecurityInfo $registrationParams = @{ EnvironmentName = $cloudRole.PublicInfo.RegistrationCloudName SubscriptionId = $cloudRole.PublicInfo.RegistrationSubscriptionId TenantId = $cloudRole.PublicInfo.RegistrationTenantId Region = $cloudRole.PublicInfo.RegistrationRegion ResourceGroupName = $cloudRole.PublicInfo.RegistrationResourceGroupName ResourceName = $cloudRole.PublicInfo.RegistrationResourceName CloudId = $cloudRole.PublicInfo.CloudId EnableAzureArcServer = [System.Convert]::ToBoolean($cloudRole.PublicInfo.EnableAzureArcServer) ComputerName = $nodeNames | Select-Object -First 1 } if (![String]::IsNullOrEmpty($cloudRole.PublicInfo.RegistrationArcServerResourceGroupName)) { Trace-Execution "Using RegistrationArcServerResourceGroupName $($cloudRole.PublicInfo.RegistrationArcServerResourceGroupName) for ArcServer registration" $registrationParams.ResourceGroupName = $cloudRole.PublicInfo.RegistrationArcServerResourceGroupName } $registrationParameterSet = $cloudRole.PublicInfo.RegistrationParameterSet if ($registrationParameterSet -eq "DefaultSet") { Trace-Execution "RegistrationParameterSet $registrationParameterSet, getting access tokens using user token cache" $registrationTokenCacheUser = $securityInfo.AADUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.RegistrationTokenCacheID $registrationTokenCacheCred = $Parameters.GetCredential($registrationTokenCacheUser.Credential) $clientId = $cloudRole.PublicInfo.RegistrationClientId Trace-Execution "Using clientId $clientId to get access token" Trace-Execution "Going to use RegistrationHelpers $cloudDeploymentNugetPath\content\Setup\Common\RegistrationHelpers.psm1" Import-Module "$cloudDeploymentNugetPath\content\Setup\Common\RegistrationHelpers.psm1" $armAccessToken = Get-AccessToken -AzureEnvironment $registrationParams.EnvironmentName -TenantId $registrationParams.TenantId -TokenCacheCred $registrationTokenCacheCred -ClientId $clientId Trace-Execution "Access tokens length using token cache is $($armAccessToken.AccessToken.Length)" $registrationParams += @{ AccountId = $registrationTokenCacheCred.UserName ArmAccessToken = $armAccessToken.AccessToken } } else { Trace-Execution "RegistrationParameterSet $registrationParameterSet, getting access tokens using service principal" $registrationSPUser = $securityInfo.AADUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.RegistrationSPID $registrationSPCred = $Parameters.GetCredential($registrationSPUser.Credential) Login-AzAccount -Environment $registrationParams.EnvironmentName -Credential $registrationSPCred -Tenant $registrationParams.TenantId -ServicePrincipal $armAccessToken = Get-AzAccessToken -Verbose Trace-Execution "Access token length using service principal is $($armAccessToken.Token.Length)" $registrationParams += @{ AccountId = $registrationSPCred.UserName ArmAccessToken = $armAccessToken.Token } } if($registrationParams.ArmAccessToken.Length -eq 0) { throw "GetRegistrationParameters failed to retrieve AccessToken" } return $registrationParams } static [string] GetProxyUrl([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "Checking if proxy settings exist" $proxySettings = Get-ASProxySettings -Parameters $Parameters $proxyUrl = $proxySettings.HTTP if ($proxyUrl) { if (-not (($proxyUrl -cmatch "http://") -or ($proxyUrl -cmatch "https://"))) { $proxyUrl = "http://$proxyUrl" } } if ($proxyUrl) { Write-ObservabilityLog "Using proxy url $proxyUrl for Arc Agent connection." } return $proxyUrl } } # SIG # Begin signature block # MIIoPAYJKoZIhvcNAQcCoIIoLTCCKCkCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCBBpx43kCkhtksG # uDg/9NEFdCguCGbemzw/vGOo0Ps6+aCCDYUwggYDMIID66ADAgECAhMzAAADri01 # UchTj1UdAAAAAAOuMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMxMTE2MTkwODU5WhcNMjQxMTE0MTkwODU5WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQD0IPymNjfDEKg+YyE6SjDvJwKW1+pieqTjAY0CnOHZ1Nj5irGjNZPMlQ4HfxXG # yAVCZcEWE4x2sZgam872R1s0+TAelOtbqFmoW4suJHAYoTHhkznNVKpscm5fZ899 # QnReZv5WtWwbD8HAFXbPPStW2JKCqPcZ54Y6wbuWV9bKtKPImqbkMcTejTgEAj82 # 6GQc6/Th66Koka8cUIvz59e/IP04DGrh9wkq2jIFvQ8EDegw1B4KyJTIs76+hmpV # M5SwBZjRs3liOQrierkNVo11WuujB3kBf2CbPoP9MlOyyezqkMIbTRj4OHeKlamd # WaSFhwHLJRIQpfc8sLwOSIBBAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUhx/vdKmXhwc4WiWXbsf0I53h8T8w # VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzUwMTgzNjAfBgNVHSMEGDAW # gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw # MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx # XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB # AGrJYDUS7s8o0yNprGXRXuAnRcHKxSjFmW4wclcUTYsQZkhnbMwthWM6cAYb/h2W # 5GNKtlmj/y/CThe3y/o0EH2h+jwfU/9eJ0fK1ZO/2WD0xi777qU+a7l8KjMPdwjY # 0tk9bYEGEZfYPRHy1AGPQVuZlG4i5ymJDsMrcIcqV8pxzsw/yk/O4y/nlOjHz4oV # APU0br5t9tgD8E08GSDi3I6H57Ftod9w26h0MlQiOr10Xqhr5iPLS7SlQwj8HW37 # ybqsmjQpKhmWul6xiXSNGGm36GarHy4Q1egYlxhlUnk3ZKSr3QtWIo1GGL03hT57 # xzjL25fKiZQX/q+II8nuG5M0Qmjvl6Egltr4hZ3e3FQRzRHfLoNPq3ELpxbWdH8t # Nuj0j/x9Crnfwbki8n57mJKI5JVWRWTSLmbTcDDLkTZlJLg9V1BIJwXGY3i2kR9i # 5HsADL8YlW0gMWVSlKB1eiSlK6LmFi0rVH16dde+j5T/EaQtFz6qngN7d1lvO7uk # 6rtX+MLKG4LDRsQgBTi6sIYiKntMjoYFHMPvI/OMUip5ljtLitVbkFGfagSqmbxK # 7rJMhC8wiTzHanBg1Rrbff1niBbnFbbV4UDmYumjs1FIpFCazk6AADXxoKCo5TsO # zSHqr9gHgGYQC2hMyX9MGLIpowYCURx3L7kUiGbOiMwaMIIHejCCBWKgAwIBAgIK # YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm # aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw # OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD # VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG # 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la # UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc # 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D # dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ # lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk # kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 # A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd # X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL # 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd # sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 # T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS # 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI # bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL # BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD # uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF # BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h # cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA # YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn # 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 # v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b # pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ # KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy # CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp # mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi # hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb # BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS # oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL # gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX # cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGg0wghoJAgEBMIGVMH4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p # Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAAOuLTVRyFOPVR0AAAAA # A64wDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIGb5 # nbC0liyCvJvvGHIl7YVEyAtoo3iryOChJVbs8p4eMEIGCisGAQQBgjcCAQwxNDAy # oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20wDQYJKoZIhvcNAQEBBQAEggEAakjLe08/eridaMrxb2+DR783QF22TSZHp5XJ # KxM/KJIAiwa78oMJLM1gS18s3UL7cImyJS9xpDp3JfjJDy6KJyl1phKBYACNsYXr # jtPLgfzpP7fXeflb4XhJNz2pO0QjDcT+vm3R/sw7uy1/bjXZF/9Tqrz+bR1kNOA8 # AnLa7VmqRT7JBL9imW9CMzxfAwQbAUu7xPxtXhDgRCBdmmY2mqQGtEgaHievi7yZ # 0KU76kEavcNkYiNEgv09ZE8uPHdEUyLUShMzpRKwZwV9X6Hsg8sMBPGHk2YMxZQF # NuoMoxULHf0jJE28NfF3JCe/5SnKyUnRJI7wAdIn1qVAgFzJPaGCF5cwgheTBgor # BgEEAYI3AwMBMYIXgzCCF38GCSqGSIb3DQEHAqCCF3AwghdsAgEDMQ8wDQYJYIZI # AWUDBAIBBQAwggFSBgsqhkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGE # WQoDATAxMA0GCWCGSAFlAwQCAQUABCBSNQbBcjolOt4RlNqkoFFFGXBrtqtfJ4EF # ArVnXsfJyQIGZeeoQKyZGBMyMDI0MDMxMTE4MTgyMi4xMTRaMASAAgH0oIHRpIHO # MIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQL # ExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxk # IFRTUyBFU046RTAwMi0wNUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1l # LVN0YW1wIFNlcnZpY2WgghHtMIIHIDCCBQigAwIBAgITMwAAAe4F0wIwspqdpwAB # AAAB7jANBgkqhkiG9w0BAQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz # aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv # cnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAx # MDAeFw0yMzEyMDYxODQ1NDRaFw0yNTAzMDUxODQ1NDRaMIHLMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1l # cmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046RTAwMi0w # NUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2Uw # ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC+8byl16KEia8xKS4vVL7R # EOOR7LzYCLXEtWgeqyOVlrzuEz+AoCa4tBGESjbHTXECeMOwP9TPeKaKalfTU5XS # GjpJhpGx59fxMJoTYWPzzD0O2RAlyBmOBBmiLDXRDQJL1RtuAjvCiLulVQeiPI8V # 7+HhTR391TbC1beSxwXfdKJqY1onjDawqDJAmtwsA/gmqXgHwF9fZWcwKSuXiZBT # bU5fcm3bhhlRNw5d04Ld15ZWzVl/VDp/iRerGo2Is/0Wwn/a3eGOdHrvfwIbfk6l # VqwbNQE11Oedn2uvRjKWEwerXL70OuDZ8vLzxry0yEdvQ8ky+Vfq8mfEXS907Y7r # N/HYX6cCsC2soyXG3OwCtLA7o0/+kKJZuOrD5HUrSz3kfqgDlmWy67z8ZZPjkiDC # 1dYW1jN77t5iSl5Wp1HKBp7JU8RiRI+vY2i1cb5X2REkw3WrNW/jbofXEs9t4bgd # +yU8sgKn9MtVnQ65s6QG72M/yaUZG2HMI31tm9mooH29vPBO9jDMOIu0LwzUTkIW # flgd/vEWfTNcPWEQj7fsWuSoVuJ3uBqwNmRSpmQDzSfMaIzuys0pvV1jFWqtqwwC # caY/WXsb/axkxB/zCTdHSBUJ8Tm3i4PM9skiunXY+cSqH58jWkpHbbLA3Ofss7e+ # JbMjKmTdcjmSkb5oN8qU1wIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFBCIzT8a2dwg # nr37xd+2v1/cdqYIMB8GA1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8G # A1UdHwRYMFYwVKBSoFCGTmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMv # Y3JsL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBs # BggrBgEFBQcBAQRgMF4wXAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0 # LmNvbS9wa2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUy # MDIwMTAoMSkuY3J0MAwGA1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUH # AwgwDgYDVR0PAQH/BAQDAgeAMA0GCSqGSIb3DQEBCwUAA4ICAQB3ZyAva2EKOWSV # pBnYkzX8f8GZjaOs577F9o14Anh9lKy6tS34wXoPXEyQp1v1iI7rJzZVG7rpUzna # y2n9csfn3p6y7kYkHqtSugCGmTiiBkwhFfSByKPI08MklgvJvKTZb673yGfpFwPj # QwZeI6EPj/OAtpYkT7IUXqMki1CRMJKgeY4wURCccIujdWRkoVv4J3q/87KE0qPQ # mAR9fqMNxjI3ZClVxA4wiM3tNVlRbF9SgpOnjVo3P/I5p8Jd41hNSVCx/8j3qM7a # LSKtDzOEUNs+ZtjhznmZgUd7/AWHDhwBHdL57TI9h7niZkfOZOXncYsKxG4gryTs # hU6G6sAYpbqdME/+/g1uer7VGIHUtLq3W0Anm8lAfS9PqthskZt54JF28CHdsFq/ # 7XVBtFlxL/KgcQylJNnia+anixUG60yUDt3FMGSJI34xG9NHsz3BpqSWueGtJhQ5 # ZN0K8ju0vNVgF+Dv05sirPg0ftSKf9FVECp93o8ogF48jh8CT/B32lz1D6Truk4E # zcw7E1OhtOMf7DHgPMWf6WOdYnf+HaSJx7ZTXCJsW5oOkM0sLitxBpSpGcj2YjnN # znCpsEPZat0h+6d7ulRaWR5RHAUyFFQ9jRa7KWaNGdELTs+nHSlYjYeQpK5QSXji # gdKlLQPBlX+9zOoGAJhoZfrpjq4nQDCCB3EwggVZoAMCAQICEzMAAAAVxedrngKb # SZkAAAAAABUwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQI # EwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3Nv # ZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmlj # YXRlIEF1dGhvcml0eSAyMDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIy # NVowfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcT # B1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UE # AxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXI # yjVX9gF/bErg4r25PhdgM/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjo # YH1qUoNEt6aORmsHFPPFdvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1y # aa8dq6z2Nr41JmTamDu6GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v # 3byNpOORj7I5LFGc6XBpDco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pG # ve2krnopN6zL64NF50ZuyjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viS # kR4dPf0gz3N9QZpGdc3EXzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYr # bqgSUei/BQOj0XOmTTd0lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlM # jgK8QmguEOqEUUbi0b1qGFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSL # W6CmgyFdXzB0kZSU2LlQ+QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AF # emzFER1y7435UsSFF5PAPBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIu # rQIDAQABo4IB3TCCAdkwEgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIE # FgQUKqdS/mTEmr6CkTxGNSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWn # G1M1GelyMFwGA1UdIARVMFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEW # M2h0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5 # Lmh0bTATBgNVHSUEDDAKBggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBi # AEMAQTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV # 9lbLj+iiXGJo0T2UkFvXzpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3Js # Lm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAx # MC0wNi0yMy5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2 # LTIzLmNydDANBgkqhkiG9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv # 6lwUtj5OR2R4sQaTlz0xM7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZn # OlNN3Zi6th542DYunKmCVgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1 # bSNU5HhTdSRXud2f8449xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4 # rPf5KYnDvBewVIVCs/wMnosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU # 6ZGyqVvfSaN0DLzskYDSPeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDF # NLB62FD+CljdQDzHVG2dY3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/ # HltEAY5aGZFrDZ+kKNxnGSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdU # CbFpAUR+fKFhbHP+CrvsQWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKi # excdFYmNcP7ntdAoGokLjzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTm # dHRbatGePu1+oDEzfbzL6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZq # ELQdVTNYs6FwZvKhggNQMIICOAIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJp # Y2EgT3BlcmF0aW9uczEnMCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOkUwMDItMDVF # MC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMK # AQEwBwYFKw4DAhoDFQCIo6bVNvflFxbUWCDQ3YYKy6O+k6CBgzCBgKR+MHwxCzAJ # BgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25k # MR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jv # c29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA6ZlmfjAi # GA8yMDI0MDMxMTExMTYxNFoYDzIwMjQwMzEyMTExNjE0WjB3MD0GCisGAQQBhFkK # BAExLzAtMAoCBQDpmWZ+AgEAMAoCAQACAiBaAgH/MAcCAQACAhOjMAoCBQDpmrf+ # AgEAMDYGCisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSCh # CjAIAgEAAgMBhqAwDQYJKoZIhvcNAQELBQADggEBAG2ZeoCneJTVRJr8Eotj4iIN # TFIY+w3HKyCD+D+C5yfTDmialNad17rp8mU7VXQYfnJUd2XrUgxfDtcE4HzGjXD/ # b5vz4kVGO19nVWPPqU4qSih5weF35OvFIi4nERAJ62E9MkK30ZkXs0+sqjahztdM # yt32ujEnFKnmsmCuopEMSM0f+38olXFSL2ebINDMm4GtCRq/F4pMyGgsAs4ovmxE # kFbU3to66BOrW0V/fHuEM7PEwtEhaTOnZVoYTnSwtz7YqOMnR6+xr7er934J62DN # iRUl9gGGzCGBAS2blfWEcF65Ao8+Yn4XO+UVU1Ux+HpetNAbEE6a522iZXT8b9kx # ggQNMIIECQIBATCBkzB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3Rv # bjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0 # aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAA # Ae4F0wIwspqdpwABAAAB7jANBglghkgBZQMEAgEFAKCCAUowGgYJKoZIhvcNAQkD # MQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJBDEiBCDk46kDz/yVjBHDDtQbYx4O # rnonBRzwXX1ts6PjFdArkjCB+gYLKoZIhvcNAQkQAi8xgeowgecwgeQwgb0EIE9Q # dxSVhfq+Vdf+DPs+5EIkBz9oCS/OQflHkVRhfjAhMIGYMIGApH4wfDELMAkGA1UE # BhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAc # BgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0 # IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAHuBdMCMLKanacAAQAAAe4wIgQgcc0i # ipYuecHTORDGPzOAJbPJic970v2yGbGx3xdP7yYwDQYJKoZIhvcNAQELBQAEggIA # Vfo+MnUMXGtMv4P2S2NnPfjRLHilb+vSoN5LcM91V85QsckFCNBl4aGvSMSJ41tf # Manrx8Lb9lIjZxpkH9QCHtjBRLZOXGCFGW9TUt5VerODOzuDBRL1oGbwmka4eiCf # wwMvkQ7I7JhkUGhHBHUOPdiSoWeyK4Mm3AwPtoqdxhEnswV7ro0KbngwpKiFYUCz # AR5XaSgQw1qO6BaFEQRcHxmjXvbbZeKkKUjTVoK5ibBd5YjvcJLoRJkFY9pvx8fU # vjDLzEb3CUrEBdZaANQr0KG2xiRGWIRjg84HvTPQTPrWlF2ojKwv3z/qeLuDE83s # w/ayJm3+BeiZbefj9Jpvy18Dxsd8lR0GzArzjwuK1i3TQORpEbTUM4LPjvjWf4iQ # 3/uRxRzViKwp7AVvluGM53IDNpid/z+Q71W6S0SapyD50jt5HqzQlc1d6+Da1OCk # 897ZI4jnGiUQdOx5l4M45X1Ojvz+UPzLU9dRUzRvLo3/1Z2VdtZvDcFzfwjAYM4I # 3gj8NFoRZa8f7L+llmqsNgLI2XfjaZ1/2nRMkVi4bEztJAWyGaiSfCCTGH4EFp1n # rWYWFLlP5AcqienMxq6rqy1Ij3QUWFjU1Ci7IfMOrIAj/YBRPqa/GoseFhF6VhNr # /R4BHq8GdlCxOcOAxgduYHHxxWC44i9YwQH7O2qleEg= # SIG # End signature block |