AzStackHci.EnvironmentChecker

1.2100.2410.204

Obs/bin/ObsDep/content/Powershell/ObservabilityConfig.psm1

<###################################################
# #
# Copyright (c) Microsoft. All rights reserved. #
# #
##################################################>

class ObservabilityConfig
{
    static RegisterObservabilityEventSource([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        $eventSources = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EventSources.EventSource

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        foreach ($eventSource in $eventSources)
        {
            $eventSourceRootName = $eventSource.EventSourceRootName
            $eventSourceLogName = $eventSource.EventSourceLogName
            $eventSourceLogSizeInBytes = $eventSource.EventSourceLogSizeInBytes
            $eventSourcePublisher = $eventSource.EventSourcePublisher

            Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog"
            try
            {
                # Register eventsource on hosts
                Invoke-Command `
                    -ComputerName $hostIps `
                    -Credential $localAdminCredential `
                    -Authentication Credssp `
                    -ScriptBlock {
                        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                        $manifestDllPath = "$observabilityNugetPath\lib\net472\$Using:eventSourceRootName.dll"
                        $manifestPath = "$observabilityNugetPath\lib\net472\$Using:eventSourceRootName.man"

                        # Grant read only access to manifest files
                        Trace-Execution "Granting read only access to $manifestPath"
                        icacls $manifestPath /grant Everyone:R
                        Trace-Execution "Granting read only access to $manifestDllPath"
                        icacls $manifestDllPath /grant Everyone:R


                        $publisherList = wevtutil ep
                        if ($publisherList.Contains($Using:eventSourcePublisher))
                        {
                            Trace-Execution "Publisher $Using:eventSourcePublisher already exists. Uninstalling."
                            wevtutil uninstall-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath"
                            Trace-Execution "Successfully uninstalled publisher $Using:eventSourcePublisher."
                        }

                        # Register the EventSource with Windows
                        Trace-Execution "wevtutil installing manifest $manifestPath with resourceFilePath and messageFilePath $manifestDllPath"
                        wevtutil install-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath"

                        Trace-Execution "wevtutil setting log size to $Using:eventSourceLogSizeInBytes"
                        wevtutil set-log $Using:eventSourceLogName /MaxSize:"$Using:eventSourceLogSizeInBytes"
                    }
            }
            catch
            {
                Trace-Execution "Registering Observability EventSource failed with error: $_"
                throw $_
            }

            Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog suceeded."
        }
    }

    static CreateObservabilityVolume([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $volumeLabel = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityVolumeLabel
        $volumeSize = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveSizeInBytes

        $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveAccessPath.Path
        $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath)

        $PhysicalDriveLetter = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.PhysicalDriveLetter
        $volumeFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveFileName
        $volumeLocation = "${PhysicalDriveLetter}:"
        $volumePath = Join-Path -Path $volumeLocation -ChildPath $volumeFileName

        Write-ObservabilityVolumeCreationStartTelemetry `
            -ComputerNames ($hostIps -join " ") `
            -VolumeFilePath $volumePath `
            -VolumeAccessPath $driveAccessPath `
            -VolumeLabel $volumeLabel `
            -VolumeSize $volumeSize

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        # Add Observability Volume to each Host
        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $VolumePath,

                [string]
                [parameter(Mandatory=$true)]
                $AccessPath,

                [string]
                [parameter(Mandatory=$true)]
                $VolumeLocation,

                [string]
                [Parameter(Mandatory=$true)]
                $VolumeLabel,

                [string]
                [Parameter(Mandatory=$true)]
                $VolumeSize
            )
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
            Add-ObservabilityVolumeWithRetry -Path $VolumePath -AccessPath $AccessPath -VolumeLabel $VolumeLabel -Size $VolumeSize -StaticSize
            Add-VolumeAccessPath -AccessPath $AccessPath -VolumePath $VolumePath
            Add-MountVolumeScheduledTask -Path $VolumePath
        }

        $argList = @($volumePath, $driveAccessPath, $volumeLocation, $volumeLabel, $volumeSize)
        Invoke-Command `
            -ComputerName $hostIps `
            -ScriptBlock $scriptBlock `
            -ArgumentList $argList `
            -Credential $localAdminCredential `
            -Authentication Credssp
        Write-ObservabilityLog "Observability Volume Setup Succeeded"
    }

    static SetUptimeScheduledTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityUptimeHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        Write-InfoLog "Obtaining local admin credential..."
        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        Write-InfoLog "Starting Uptime scheduled task Setup"
        Write-InfoLog "Hosts: $hostNames"
        $scriptBlock = {
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityUptimeHelpers.psm1"
            $hostName = $env:COMPUTERNAME
            Write-InfoLog "Starting uptime scheduled task on host $hostName."
            try
            {
                Set-UptimeTaskWithRetry
                Write-InfoLog "Uptime scheduled task on host $hostName succeeded."
            }
            catch
            {
                Write-ErrorLog "Uptime scheduled task on host $hostName failed with exception $_ ."
                throw $_
            }
        }

        Invoke-Command -ComputerName $hostIps -ScriptBlock $scriptBlock -Credential $localAdminCredential -Authentication Credssp | Out-Null
        Write-InfoLog "Uptime scheduled task completed."
    }

    static SetObservabilityEventTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters, $eventName, $eventScriptBlock)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        Write-InfoLog "Starting $eventName events task Setup"
        Write-InfoLog "Hosts: $hostNames"
        $exceptionMessage = ""
        $scriptBlock = $eventScriptBlock

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        foreach ($hostIp in $hostIps)
        {
            Write-InfoLog "Starting $eventName events task on host $hostIp."
            $result = Invoke-Command -ComputerName $hostIp -ScriptBlock $scriptBlock -Credential $localAdminCredential -Authentication Credssp

            if($result -eq $true)
            {
                Write-InfoLog "$eventName events task on host $hostIp succeeded."
            }
            else
            {
                Write-ErrorLog "$eventName events task on host $hostIp failed with exception $result."
                $exceptionMessage += "${hostIp}: $result`n"
            }
        }

        if($exceptionMessage)
        {
            throw $exceptionMessage
        }
    }

    static SetCensusEventScheduledTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1"
        $scriptBlock = {
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1"
            Set-CensusTaskWithRetry
        }

        [ObservabilityConfig]::SetObservabilityEventTask($Parameters, "CensusEvent", $scriptBlock)

        Write-InfoLog "Census events scheduled task completed."
    }

    static SetRegistrationEventOneTimeTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1"
        $scriptBlock = {
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1"
            Set-RegistrationEventsTaskWithRetry
        }

        [ObservabilityConfig]::SetObservabilityEventTask($Parameters, "RegistrationEvents", $scriptBlock)

        Write-InfoLog "Registration events one-time task completed."
    }

    static CreateVolumeFoldersAndPruner([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $folderCleanupThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaCleanupThresholdInPercent
        $folderFreeSpaceThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaFreeSpaceThresholdInPercent
        $purgeFolderFrequencyInMinutes = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.PurgeFolderFrequencyInMinutes
        $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilitySubFolderConfigFileName
        $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveAccessPath.Path
        $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath)

        Write-ObservabilityLog "Starting Observability Volume Folder and Pruner Setup"
        Write-ObservabilityLog "Hosts: $hostNames"
        Write-ObservabilityLog "Volume folder mount access path: $driveAccessPath"
        Write-ObservabilityLog "Observability subfolder folder cleanup threshold: $folderCleanupThresholdPercent"
        Write-ObservabilityLog "Observability subfolder folder free space threshold: $folderFreeSpaceThresholdPercent"
        Write-ObservabilityLog "Observability volume purge frequency in minutes: $purgeFolderFrequencyInMinutes"

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        # Add Observability Volume to each Host
        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $AccessPath,

                [string]
                [Parameter(Mandatory=$true)]
                $FolderCleanupThresholdPercent,

                [string]
                [Parameter(Mandatory=$true)]
                $FolderFreeSpaceThresholdPercent,

                [string]
                [Parameter(Mandatory=$true)]
                $PurgeFolderFrequencyInMinutes,

                [string]
                [Parameter(Mandatory=$true)]
                $SubFolderConfigFileName
            )
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
            New-VolumeFoldersAndPrunerWithRetry `
                -AccessPath $AccessPath `
                -CleanupThresholdPercent $FolderCleanupThresholdPercent `
                -FreeSpaceThresholdPercent $FolderFreeSpaceThresholdPercent `
                -PurgeFolderFrequencyInMinutes $PurgeFolderFrequencyInMinutes `
                -SubFolderConfigFileName $SubFolderConfigFileName
        }

        $argList = @(
            $driveAccessPath,
            $folderCleanupThresholdPercent,
            $folderFreeSpaceThresholdPercent,
            $purgeFolderFrequencyInMinutes,
            $subFolderConfigFileName
        )
        Invoke-Command `
            -ComputerName $hostIps `
            -ScriptBlock $scriptBlock `
            -ArgumentList $argList `
            -Credential $localAdminCredential `
            -Authentication Credssp
        Write-ObservabilityLog "Observability subfolder and pruner Setup Succeeded"
    }

    # Set registry key HKLM\Software\Microsoft\AzureStack DeviceType to AzureEdge
    static SetDeviceTypeRegistryKey([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
        Write-ObservabilityLog "SetDeviceTypeRegistryKey start."

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        try
        {
            Invoke-Command `
                -ComputerName $hostIps `
                -Credential $localAdminCredential `
                -Authentication Credssp `
                -ScriptBlock {
                    Write-Verbose "Setting HKLM\Software\Microsoft\AzureStack DeviceType registry key to AzureEdge."
                    $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\'
                    if (!(Test-Path -Path $registryPath))
                    {
                        New-Item -Path $registryPath -Force
                    }
                    New-ItemProperty -Path $registryPath -Name 'DeviceType' -PropertyType 'String' -Value "AzureEdge" -Force
                    Write-Verbose "Finished DeviceType Registry key setup on $($env:COMPUTERNAME)"
                }
            Write-ObservabilityLog "Succeeded in setting AzureStack DeviceType Registry key."
        }
        catch
        {
            Write-ObservabilityErrorLog "[Error] DeviceType registry key set up failed with an error : $_ "
            throw $_
        }
    }

    # Set up AzureStack environment to enable Autologger and UTC Telemetry via MA
    # Reg key set up needs to happend upfornt during host configuration
    static SetUpUtcExporterFeature([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        Write-ObservabilityLog "Set up AzureStack environment to enable Autologger and UTC Telemetry via MA."

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        try
        {
            Invoke-Command `
                -ComputerName $hostIps `
                -Credential $localAdminCredential `
                -Authentication Credssp `
                -ScriptBlock {
                    $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                    $utcExporterPath = "$($env:SystemRoot)\System32\UtcExporters"
                    # Create UtcExporters folder in system32
                    if (-not (Test-Path -Path $utcExporterPath))
                    {
                        New-Item -Path "$($env:SystemRoot)\System32" -ItemType Directory -Name 'UtcExporters' -Force
                    }

                    Write-Verbose "Set up the required regkey to enable utc exporter Feature"
                    New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack' -Name 'AllowExporters' -PropertyType 'DWORD' -Value 1 -Force
                    New-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters' -Force
                    New-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Force
                    New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Name 'DllPath' -PropertyType 'String' -Value "$($utcExporterPath)\UtcGenevaExporter.dll" -Force
                    New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Name 'GenevaNamespace' -PropertyType 'String' -Value "AEOppeTely" -Force
                    New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\TestHooks' -Name 'SkipSignatureMitigation' -PropertyType 'DWORD' -Value 1 -Force

                    # Update UTC Service binary
                    Write-Verbose "Updating UTC service dll"
                    Stop-Service "diagtrack" -Force
                    Start-Sleep -Seconds 10
                    Write-Verbose "Copy UTC Exporter dll to $($utcExporterPath)"
                    Copy-Item -Path "$observabilityNugetPath\lib\net472\UtcGenevaExporter.dll" -Destination $utcExporterPath -Force
                    Start-Service "diagtrack"

                    Write-Verbose "Finished UTC Exporter setup on $($env:COMPUTERNAME)"
                }
            Write-ObservabilityLog "Set up AzureStack environment to enable Autologger and UTC Telemetry via MA succeeded."
        }
        catch
        {
            Write-ObservabilityErrorLog "[Error] UTC Exporter set up failed with an error : $_ "
            throw $_
        }
    }

    # Install VC Runtime on all hosts
    static InstallVcRuntime([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }
        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }
        Write-ObservabilityLog "Starting installation of X64 VC Redistributable."

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        try
        {
            Invoke-Command `
                -ComputerName $hostIps `
                -Credential $localAdminCredential `
                -Authentication Credssp `
                -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

                Write-ObservabilityLog "[Starting] installation of VC Redistributable on host $($env:COMPUTERNAME)"
                $vcRedistFilePath = "$observabilityNugetPath\content\VS17\VC_redist.x64.exe"
                if(Test-Path -Path $vcRedistFilePath) {
                    Start-Process -File $vcRedistFilePath -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow
                } else {
                    $errMsg = "[ERROR] VC Redistributable [$vcRedistilePath] not found on host $($env:COMPUTERNAME)"
                    Write-ObservabilityErrorLog $errMsg
                    throw $errMsg
                }
                Write-ObservabilityLog "[Finished] installation of VC Redistributable on host $($env:COMPUTERNAME)"
            }
        }
        catch
        {
            Write-ObservabilityErrorLog "[Failed] installation of VC Redistributable: $_ "
            throw $_
        }
        Write-ObservabilityLog "Finished installation of X64 VC Redistributable."
    }

    static CreateObservabilityVolumeOnNC([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "Starting observability volume setup on NC VMs"
        $ncVMs = $Parameters.Roles["NC"].PublicConfiguration.Nodes.Node.Name
        $volumeFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveFileName
        $volumeLabel = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityVolumeLabel
        $volumeSize = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveSizeInBytes
        $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveAccessPath.Path
        $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath)

        $folderCleanupThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaCleanupThresholdInPercent
        $folderFreeSpaceThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaFreeSpaceThresholdInPercent
        $purgeFolderFrequencyInMinutes = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.PurgeFolderFrequencyInMinutes
        $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilitySubFolderConfigFileName
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $VolumePath,

                [string]
                [parameter(Mandatory=$true)]
                $HostComputerName,

                [string]
                [parameter(Mandatory=$true)]
                $AccessPath,

                [string]
                [parameter(Mandatory=$true)]
                $FolderCleanupThresholdPercent,

                [string]
                [parameter(Mandatory=$true)]
                $FolderFreeSpaceThresholdPercent,

                [string]
                [Parameter(Mandatory=$true)]
                $PurgeFolderFrequencyInMinutes,

                [string]
                [parameter(Mandatory=$true)]
                $SubFolderConfigFileName,

                [PSCredential]
                [parameter(Mandatory=$true)]
                $Credential
            )
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

            $remoteVolumePath = "\\$HostComputerName\$VolumePath"
            $remoteVolumePath = $remoteVolumePath -replace ":", "$"
            Add-VolumeAccessPath -AccessPath $AccessPath -VolumePath $remoteVolumePath
            Add-MountVolumeScheduledTask -Path $remoteVolumePath -Credential $Credential
            New-VolumeFoldersAndPrunerWithRetry `
                -AccessPath $AccessPath `
                -CleanupThresholdPercent $FolderCleanupThresholdPercent `
                -FreeSpaceThresholdPercent $FolderFreeSpaceThresholdPercent `
                -PurgeFolderFrequencyInMinutes $PurgeFolderFrequencyInMinutes `
                -SubFolderConfigFileName $SubFolderConfigFileName
            Set-FolderQuotas -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName
        }

        if(-not($ncVMs -is [System.Array])) # Only 1 VM exists
        {
            $ncVMs = @($ncVMs)
        }
        foreach($ncVM in $ncVMs)
        {
            $hostComputerName = $env:COMPUTERNAME
            Write-ObservabilityLog "Starting observability volume onboarding for $ncVM."
            try
            {
                Get-VM $ncVM | Out-Null # Test that VM is accessbile
            }
            catch
            {
                Write-ObservabilityLog "$ncVM not found on $hostComputerName. No op."
                continue
            }
            $vmHardDrivePath = (Get-VMHardDiskDrive $ncVM).Path
            if($vmHardDrivePath -is [System.Array])
            {
                $vmHardDrivePath = $vmHardDrivePath[0]
            }
            $volumeLocation = Split-Path $vmHardDrivePath -Parent
            $volumePath = Join-Path -Path $volumeLocation -ChildPath $volumeFileName
            Write-ObservabilityLog "Adding observability volume at volume path $volumePath"
            Add-ObservabilityVolumeWithRetry -Path $volumePath -AccessPath $driveAccessPath -VolumeLabel $volumeLabel -Size $volumeSize -StaticSize
            if((Get-DiskImage $volumePath).Attached)
            {
                Dismount-VHD $volumePath
            }
            $argList = @(
                $volumePath,
                $hostComputerName,
                $driveAccessPath,
                $folderCleanupThresholdPercent,
                $folderFreeSpaceThresholdPercent,
                $purgeFolderFrequencyInMinutes,
                $subFolderConfigFileName,
                $domainUserCredential
            )
            Invoke-Command `
                -ComputerName $ncVM `
                -ScriptBlock $scriptBlock `
                -Credential $domainUserCredential `
                -Authentication Credssp `
                -ArgumentList $argList
            Write-ObservabilityLog "Finished observability volume onboarding for $ncVM."
        }
    }

    static RegisterObservabilityEventSourceOnNC([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $eventSources = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EventSources.EventSource
        $ncVMs = $Parameters.Roles["NC"].PublicConfiguration.Nodes.Node.Name

        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        foreach ($eventSource in $eventSources)
        {
            $eventSourceRootName = $eventSource.EventSourceRootName
            $eventSourceLogName = $eventSource.EventSourceLogName
            $eventSourceLogSizeInBytes = $eventSource.EventSourceLogSizeInBytes

            Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog"
            $scriptBlock = {
                param (
                    [string]
                    [parameter(Mandatory=$true)]
                    $EventSourceRootName,

                    [string]
                    [parameter(Mandatory=$true)]
                    $EventSourceLogName,

                    [string]
                    [parameter(Mandatory=$true)]
                    $EventSourceLogSizeInBytes
                )
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                $manifestDllPath = "$observabilityNugetPath\lib\net472\$EventSourceRootName.dll"
                $manifestPath = "$observabilityNugetPath\lib\net472\$EventSourceRootName.man"

                # Grant read only access to manifest files
                Trace-Execution "Granting read only access to $manifestPath"
                icacls $manifestPath /grant Everyone:R
                Trace-Execution "Granting read only access to $manifestDllPath"
                icacls $manifestDllPath /grant Everyone:R

                # Register the EventSource with Windows
                Trace-Execution "wevtutil installing manifest $manifestPath with resourceFilePath and messageFilePath $manifestDllPath"
                wevtutil install-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath"

                Trace-Execution "wevtutil setting log size to $EventSourceLogSizeInBytes"
                wevtutil set-log $EventSourceLogName /MaxSize:"$EventSourceLogSizeInBytes"
            }

            try
            {
                # Register eventsource on hosts
                Invoke-Command `
                    -ComputerName $ncVMs `
                    -ScriptBlock $scriptBlock `
                    -Credential $domainUserCredential `
                    -Authentication Credssp `
                    -ArgumentList @($eventSourceRootName, $eventSourceLogName, $eventSourceLogSizeInBytes)
            }
            catch
            {
                Trace-Execution "Registering Observability EventSource failed with error: $_"
                throw $_
            }

            Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog suceeded."
        }
    }

    static SetFolderQuotas ([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilitySubFolderConfigFileName
        $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveAccessPath.Path
        $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath)

        Write-ObservabilityLog "Starting observability volume fsrm quota setup on $hostNames"
        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $AccessPath,

                [string]
                [parameter(Mandatory=$true)]
                $SubFolderConfigFileName
            )
            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

            Set-FolderQuotas -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName
        }
        try
        {
            Invoke-Command `
                -ComputerName $hostIps `
                -Credential $localAdminCredential `
                -Authentication Credssp `
                -ScriptBlock $scriptBlock `
                -ArgumentList @($driveAccessPath, $subFolderConfigFileName)
        }
        catch
        {
            Write-ObservabilityErrorLog "[Failed] setup of FSRM Quotas: $_ "
            throw $_
        }
    }

    static [PSCredential] GetLocalCredential([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "Obtaining local admin credential..."
        $localAdminCredential = $null
        try
        {
            $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
            $securityInfo = $cloudRole.PublicInfo.SecurityInfo
            $localAdmin = $securityInfo.LocalUsers.User | Where-Object Role -eq $Parameters.Configuration.Role.PrivateInfo.Accounts.BuiltInAdminAccountID
            $localAdminCredential = $Parameters.GetCredential($localAdmin.Credential)
        }
        catch
        {
            Write-ObservabilityErrorLog "Failed to obtain local admin credentials: $_ "
            throw $_
        }
        Write-ObservabilityLog "Local admin credential obtained."

        if ($localAdminCredential.UserName -eq "removed")
        {
            Write-ObservabilityLog "The local admin credential obtained has been removed. Using domain credential instead."
            return [ObservabilityConfig]::GetDomainCredential($Parameters)
        }
        return $localAdminCredential
    }

    static [PSCredential] GetDomainCredential([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "Obtaining domain user credentials..."
        $domainUserCredential = $null
        try
        {
            $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
            $securityInfo = $cloudRole.PublicInfo.SecurityInfo
            $physicalMachinesRole = $Parameters.Configuration.Role
            $domainName = $Parameters.Roles.Domain.PublicConfiguration.PublicInfo.DomainConfiguration.Fqdn

            $domainUser = $securityInfo.DomainUsers.User | Where-Object Role -EQ $physicalMachinesRole.PrivateInfo.Accounts.DomainUserAccountID
            $domainUserCredential = $Parameters.GetCredential($domainUser.Credential)
            # Making sure that the credential is domain qualified, PowerShell Direct require credential to be domain qualified
            if($($domainUserCredential.GetNetworkCredential().Domain) -eq "")
            {
                $domainUserCredential = New-Object -TypeName PSCredential -ArgumentList "$domainName\$($domainUserCredential.UserName)", $($domainUserCredential.Password)
            }
            Write-ObservabilityLog "Domain user credentials obtained."
        }
        catch
        {
            Write-ObservabilityErrorLog "Failed to obtain domain user credentials: $_ "
            throw $_
        }
        return $domainUserCredential
    }

    static InstallBootstrapObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        $exceptionMessage = ""
        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        try
        {
            Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability installation start."

            $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration

            $cloudId = $cloudRole.PublicInfo.CloudId
            $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare

            $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters)

            $scriptBlock = {
                param (
                    [string]
                    [parameter(Mandatory=$true)]
                    $AgentMsiPath,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $AccessToken,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $SubscriptionId,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $TenantId,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $AccountId,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $ResourceGroupName,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $CloudId,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $EnvironmentName,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $Region,

                    [string]
                    [Parameter(Mandatory=$true)]
                    $StampId
                )

                $errorMessage = ""
                $setHostNICIPAddressLogFile = Join-Path -Path $env:systemdrive -ChildPath $LogFileRelativePath
                Start-Transcript -Append -Path $setHostNICIPAddressLogFile
                try
                {
                    $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                    Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

                    $ResourceName = Get-ArcResourceName
                    Write-ObservabilityLog "$env:COMPUTERNAME Going to install Arc-for-server agent..."
                    Install-ArcForServerAgent `
                        -AgentMsiPath $AgentMsiPath `
                        -AccessToken $AccessToken `
                        -SubscriptionId $SubscriptionId `
                        -TenantId $TenantId `
                        -ResourceGroupName $ResourceGroupName `
                        -EnvironmentName $EnvironmentName `
                        -Region $Region `
                        -ResourceName $ResourceName | Out-Null

                    Write-ObservabilityLog "$env:COMPUTERNAME Going to download Observability Extension..."
                    Install-ArcForServerExtensions `
                        -AccessToken $AccessToken `
                        -SubscriptionId $SubscriptionId `
                        -TenantId $TenantId `
                        -AccountId $AccountId `
                        -ResourceGroupName $ResourceGroupName `
                        -ResourceName $ResourceName `
                        -EnvironmentName $EnvironmentName `
                        -Region $Region | Out-Null

                    $hostNameHash = Get-Sha256Hash -ClearString (hostname)
                    $nodeId = "$($StampId)-$hostNameHash"
                    Write-BootstrapNodeIdAndHardwareIdHashTelemetry -BootstrapNodeId $nodeId
                }
                catch{
                    $errorMessage = $PSItem.ToString()
                    Write-ObservabilityErrorLog $errorMessage
                    throw $PSItem.Exception.Message
                }
                finally
                {
                    Write-ArcForServerInstallationStopTelemetry `
                        -ComputerName $env:COMPUTERNAME `
                        -Message "Bootstrap Observability installation end" `
                        -ExceptionDetails $errorMessage
                }
            }

            $argList = @(
                $arcForServerMsiFilePath,
                $registrationParams.ArmAccessToken,
                $registrationParams.SubscriptionId,
                $registrationParams.TenantId,
                $registrationParams.AccountId,
                $registrationParams.ResourceGroupName,
                $cloudId,
                $registrationParams.EnvironmentName,
                $registrationParams.Region,
                $registrationParams.ResourceName
            )

            Invoke-Command `
                -ComputerName $hostIps `
                -ScriptBlock $scriptBlock `
                -ArgumentList $argList `
                -Credential $localAdminCredential `
                -Authentication Credssp
        }
        catch
        {
            $errMsg = "Bootstrap Observability installation failed with $_"
            $exceptionMessage += $errMsg
            Write-ObservabilityErrorLog $errMsg
            throw
        }

        Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability installation end."
    }

    static UninstallBootstrapObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability uninstallation start."
        
        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }
        
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)
        $cloudRole      = $Parameters.Roles["Cloud"].PublicConfiguration

        $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare
        Write-ObservabilityLog "$env:COMPUTERNAME Arc For Server MSI Path: $arcForServerMsiFilePath"

        $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters)


        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $AgentMsiPath,

                [string]
                [Parameter(Mandatory=$true)]
                $AccessToken,

                [string]
                [Parameter(Mandatory=$true)]
                $SubscriptionId,

                [string]
                [Parameter(Mandatory=$true)]
                $AccountId,

                [string]
                [Parameter(Mandatory=$true)]
                $ResourceGroupName,

                [string]
                [Parameter(Mandatory=$true)]
                $CloudId
            )

            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

            # Import MiscConstants to remove GMAScenario reg key
            $gmaNugetName = "Microsoft.AzureStack.Observability.GenevaMonitoringAgent"
            $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $gmaNugetName) -ChildPath "content"
            Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking
            $gmaScenarioRegKeyPath = $MiscConstants.GMAScenarioRegKey.Path
            $gmaScenarioRegKeyName = $MiscConstants.GMAScenarioRegKey.Name
            if (Test-RegKeyExists -Path $gmaScenarioRegKeyPath -Name $gmaScenarioRegKeyName)
            {
                Remove-ItemProperty `
                    -Path $gmaScenarioRegKeyPath `
                    -Name $gmaScenarioRegKeyName `
                    -Force
                Write-ObservabilityLog "Removed registry key $gmaScenarioRegKeyName at path $gmaScenarioRegKeyPath"
            }

            Uninstall-ArcForServerExtensions `
                        -AccessToken $AccessToken `
                        -SubscriptionId $SubscriptionId `
                        -AccountId $AccountId `
                        -ResourceGroupName $ResourceGroupName `
                        -CloudId $CloudId


            $result = Uninstall-ArcForServerAgent `
                        -AgentMsiPath $AgentMsiPath `
                        -AccessToken $AccessToken
 
            return $result
        }

        # Install Arc for server agent on each Host
        $exceptionMessage = ""
        foreach($hostIp in $hostIps)
        {
            $argList = @($arcForServerMsiFilePath, $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.AccountId, $registrationParams.ResourceGroupName, $registrationParams.CloudId)
            $result = Invoke-Command `
                -ComputerName $hostIp `
                -ScriptBlock $scriptBlock `
                -ArgumentList $argList `
                -Credential $domainUserCredential `
                -Authentication Credssp
            if ($result -eq $true)
            {
                Write-ObservabilityLog "Arc for server agent setup on host $hostIp succeeded."
            }
            else
            {
                Write-ObservabilityLog "Arc for server agent setup on host $hostIp failed with exception $result."
                $exceptionMessage += "${hostIp}: $result`n"
            }
        }

        if($exceptionMessage)
        {
            throw $exceptionMessage
        }

        Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability uninstallation end."
    }

        <#
        .SYNOPSIS
        Evaluation function to identify what type of GMA configuration action to use.
        Following are the cases involved during evaluation :
            case 1 : If the interface is invoked in case of Deployment then run configuration on all nodes.
            case 2 : If the interface is invoked in case of AddNode or RepairNode then run configuration on only that node.
             
        .EXAMPLE
        [GMATenantJson]::EvaluateGMAConfigurationType($Parameters)
 
        .PARAMETER Parameters
        The ECE role parameters for this interface.
    #>
    static [CloudEngine.Actions.ConditionalActionDescription] EvaluateListenerModeConfigurationType([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        $ErrorActionPreference = "Stop"

        ## For classes in PS, we need to use the call stack to get the current executing function.
        $functionName = $(Get-PSCallStack)[0].FunctionName

        try {
            Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1

            $cloudRolePath = "ObservabilityConfig"
            $listenerModeOnAllNodes = "SetListenerModeOnAllNodes"
            $listenerModeOnOneNode = "SetListenerModeOnOneNode"
            $onAllNodesAction = [CloudEngine.Actions.ConditionalActionDescription]::CreateWithDefinedAction($cloudRolePath, $listenerModeOnAllNodes)
            $onOneNodeAction = [CloudEngine.Actions.ConditionalActionDescription]::CreateWithDefinedAction($cloudRolePath, $listenerModeOnOneNode)

            $nodeName = Get-ExecutionContextNodeName -Parameters $Parameters

            if($null -ne $nodeName)
            {
                # In add node or repair node case, ExecutionContextNodeName should have exactly one value.
                Trace-Execution "Execution context node was determined. Listener Mode will be set on $nodeName only."
                return $onOneNodeAction
            }

            # In regular deployment case, ExecutionContextNodeName will be null.
            Trace-Execution "No exceution context node was determined. Listener Mode will set on all the nodes."
            return $onAllNodesAction
        }
        catch {
            Trace-Error "$functionName :Failed to evaluate ListenerMode configuration type on node $($env:COMPUTERNAME) due to following error : $_"
            throw
        }

    }

    static StopArcExtensionObservabilityAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        $hostName = Get-ExecutionContextNodeName -Parameters $Parameters

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIp = $allHosts.Keys | ForEach-Object { if($hostName.Contains($_)) {$allHosts[$_]} }

        Invoke-Command `
            -ComputerName $hostIp `
            -Credential $domainUserCredential `
            -Authentication Credssp `
            -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

                $ObsAgentServiceName = "AzureStack Observability Agent"
                $ObsAgentServiceDisplayName = "AzureStack Arc Extension Observability Agent"
                $stopDelayInSeconds = 10

                $service = Get-Service | Where-Object {$_.Name -eq $ObsAgentServiceName -and $_.DisplayName -eq $ObsAgentServiceDisplayName }
                if($service)
                {
                    if ($service.Status -ne "Stopped")
                    {
                        Write-ObservabilityLog "Stopping service $ObsAgentServiceName with display name $ObsAgentServiceDisplayName on host $($env:COMPUTERNAME)"
                        $service | Stop-Service
                        Write-ObservabilityLog "Sleeping $stopDelayInSeconds seconds after stopping $ObsAgentServiceName before unregistering."
                        Start-Sleep -Seconds $stopDelayInSeconds
                    }
                    Write-ObservabilityLog "Unregistering service $ObsAgentServiceName."
                    sc.exe delete $ObsAgentServiceName
                    Write-ObservabilityLog "Successfully unregistered service $ObsAgentServiceDisplayName"
                }
                else
                {
                    Write-ObservabilityLog "Service $ObsAgentServiceName not found. Noop."
                }
            }
    }

    static StopRemSupAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        $hostName = Get-ExecutionContextNodeName -Parameters $Parameters

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIp = $allHosts.Keys | ForEach-Object { if($hostName.Contains($_)) {$allHosts[$_]} }

        Invoke-Command `
            -ComputerName $hostIp `
            -Credential $domainUserCredential `
            -Authentication Credssp `
            -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

                # Remote Support Service names - Arc Extension, Env Checker
                $agentServiceNames = @("Observability Remote Support Agent", "RemoteSupportAgent")
                $stopDelayInSeconds = 10
                $maxRetries = 3

                foreach ($agentServiceName in $agentServiceNames)
                {
                    $attempt = 1
                    $service = Get-Service -Name $agentServiceName -ErrorAction "SilentlyContinue"
                    if($service)
                    {
                        $serviceStatus = $service.Status
                        while ($attempt -le $maxRetries -and -not [string]::IsNullOrWhiteSpace($serviceStatus))
                        {
                            try
                            {
                                if ($serviceStatus -ne "Stopped")
                                {
                                    Write-ObservabilityLog "Stopping service $agentServiceName Status: $serviceStatus, Attempt: $attempt."
                                    $service | Stop-Service
                                    Write-ObservabilityLog "Sleeping $stopDelayInSeconds seconds after stopping $agentServiceName before unregistering, Attempt: $attempt."
                                    Start-Sleep -Seconds $stopDelayInSeconds
                                }
                                Write-ObservabilityLog "Unregistering service $agentServiceName , Attempt: $attempt."
                                sc.exe delete $agentServiceName
                                Write-ObservabilityLog "Successfully unregistered service $agentServiceName , Attempt: $attempt."
                            }
                            catch{
                                Write-ObservabilityLog "Exception caught during stopping/removing service $agentServiceName . $($_.Exception)"
                                Write-ObservabilityLog "Retrying ..."
                            }

                            $attempt = $attempt + 1
                            $service = Get-Service -Name $agentServiceName -ErrorAction "SilentlyContinue"
                            $serviceStatus = $service.Status
                        }

                        if($attempt -eq $maxRetries)
                        {
                            Write-ObservabilityErrorLog "All retries $maxRetries exhausted. $agentServiceName is not unregistered."
                            throw
                        }
                    }
                    else
                    {
                        Write-ObservabilityLog "Service $agentServiceName not found. NoOp."
                    }
                }
            }
    }

    static SetWatchdogToListenerMode([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)

        $hostName = Get-ExecutionContextNodeName -Parameters $Parameters

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIp = $allHosts.Keys | ForEach-Object { if($hostName.Contains($_)) {$allHosts[$_]} }

        Invoke-Command `
            -ComputerName $hostIp `
            -Credential $domainUserCredential `
            -Authentication Credssp `
            -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
                Write-ObservabilityLog "Setting HKLM\Software\Microsoft\AzureStack\Observability WatchdogListenerMode registry key to 1 on $($env:COMPUTERNAME)."
                $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\Observability'
                if (!(Test-Path -Path $registryPath))
                {
                    New-Item -Path $registryPath -Force
                }
                New-ItemProperty -Path $registryPath -Name 'WatchdogListenerMode' -PropertyType 'DWord' -Value 1 -Force
                Write-ObservabilityLog "Finished setting WatchdogListenerMode Registry key to DWord 1 on $($env:COMPUTERNAME)"

                $timeLimitInSeconds = 600
                $retryPeriod = 10
                $elapsed = 0

                $maHostProcess = Get-Process "MonAgentHost" -ErrorAction SilentlyContinue
                $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue
                while($maHostProcess -or $maCoreProcess)
                {
                    if($maHostProcess)
                    {
                        $pids = $maHostProcess.Id | Out-String
                        Write-ObservabilityLog "MonAgentHost Active Process Ids: `n $pids"
                    }
                    if($maCoreProcess)
                    {
                        $pids = $maCoreProcess.Id | Out-String
                        Write-ObservabilityLog "MonAgentCore Active Process Ids: `n $pids"
                    }

                    if($elapsed -gt $timeLimitInSeconds)
                    {
                        $errMsg = "WatchdogListenerMode has failed to stop Monitoring agent processes after $timeLimitInSeconds."
                        Write-ObservabilityErrorLog $errMsg
                        throw $errMsg
                    }
                    Write-ObservabilityLog "Monitoring Agent processes have not yet stopped. Retrying in $retryPeriod seconds."
                    Start-Sleep -Seconds $retryPeriod
                    $elapsed += $retryPeriod
                    $maHostProcess = Get-Process "MonAgentHost" -ErrorAction SilentlyContinue
                    $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue
                }
                Write-ObservabilityLog "MonitoringAgent has been successfully stopped after Watchdog has been set to listener mode."
            }
    }

    static SetGMATenantJsonRegistryKeys([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
        Write-ObservabilityLog "SetGMATenantJsonRegistryKeys start"

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        # Get Host IPs for all hosts
        $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario
        $hostIps = $allHosts.Keys | ForEach-Object { if($hostNames.Contains($_)) {$allHosts[$_]} }

        $gmaNugetName = "Microsoft.AzureStack.Observability.GenevaMonitoringAgent"
        $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $gmaNugetName) -ChildPath "content"
        Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking

        ## Get PhysicalDriveLetter to place GMA cache folder
        $PhysicalDriveLetter = $Parameters.Roles["GMATenantJson"].PublicConfiguration.PublicInfo.PhysicalDriveLetter
        $volumeName = "${PhysicalDriveLetter}:"

        ## GMA cache folder path
        $gmaCacheFolderName = $Parameters.Roles["GMATenantJson"].PublicConfiguration.PublicInfo.GMACacheFolderName
        $gmaCacheFolderPath = Join-Path -Path $volumeName -ChildPath $gmaCacheFolderName

        ## Determine GcsEnvironment
        $gcsEnvironment = "Prod" ## default environment

        ## Check if the reg key created for CI exists or not, if yes then change the GCSEnvironment to point to PPE.
        if ((Test-RegKeyExists -Path "HKLM:\Software\Microsoft\SQMClient\" -Name "IsCIEnv") -eq $true) {
            $gcsEnvironment = "Ppe"
        }

        $envInfoFilePath = "$GmaPackageContentPath\EnvironmentInfo.json"
        $tenantInfoContent = Get-Content $envInfoFilePath -Raw | ConvertFrom-Json
        $envInfo = $tenantInfoContent.$GcsEnvironment

        # Settings from EnvironmentInfo.json
        $gcsEndpoint = $envInfo.EndPoint
        $gcsAccount = $envInfo.Account
        $genevaConfigVersion = $envInfo.ConfigVersion

        # Settings from ECE
        $assemblyVersion = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.Version
        $registrationSubscriptionId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationSubscriptionId
        $registrationResourceGroupName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationResourceGroupName
        $stampId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationResourceName
        $gcsRegionName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationRegion
        $clusterName = $Parameters.Roles["Cluster"].PublicConfiguration.Clusters.Node.Name
        $cloudId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.CloudId
        $deviceArmResourceUri = "/Subscriptions/$registrationSubscriptionId/resourceGroups/$registrationResourceGroupName/providers/Microsoft.AzureStackHCI/clusters/$stampId"
        $osBuild = Get-OSBuildVersion
        $registrationArcResourceGroupName = $registrationResourceGroupName
        $arcResourceGroupName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationArcServerResourceGroupName

        if (![String]::IsNullOrEmpty($arcResourceGroupName)) {
            Trace-Execution "Using RegistrationArcServerResourceGroupName $arcResourceGroupName for ArcServer resource in Tenant Json"
            $registrationArcResourceGroupName = $arcResourceGroupName
        }

        $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters)
        Invoke-Command `
            -ComputerName $hostIps `
            -Credential $localAdminCredential `
            -Authentication Credssp `
            -ScriptBlock {
                $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
                Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
                $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $Using:gmaNugetName) -ChildPath "content"
                Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking

               # Will unset GMA scenario registry path at bootstrap uninstall
                Set-GMAScenarioRegistryKeyToBootstrap | Out-Null

                # For Bootstrap Scenario, nodeid is stampid-sha256(hostname)
                $hostNameHash = Get-Sha256Hash -ClearString (hostname)
                $nodeId = "$($Using:stampId)-$hostNameHash"
                $arcAgentResourceId = Get-ArcResourceId -SubscriptionId $Using:registrationSubscriptionId -ResourceGroupName $Using:registrationArcResourceGroupName -CloudId $Using:cloudId

                $configTypes = @("Telemetry", "Diagnostics", "Health", "Metrics")
                foreach($configType in $configTypes)
                {
                    $envInfo = $Using:envInfo
                    $gcsNameSpace = $envInfo.Namespaces.$configType

                    # cacheLocalPath will be overwritten by Extension Install script because cache location for bootstrap is not known until install time
                    $cacheLocalPath = Join-Path -Path $Using:gmaCacheFolderPath -ChildPath $($configType + "Cache")
                    Set-TenantConfigRegistryKeys `
                        -ConfigType $configType `
                        -Version "1.0" `
                        -GcsAuthIdType "AuthMSIToken" `
                        -GcsEnvironment $Using:gcsEndpoint `
                        -GcsGenevaAccount $Using:gcsAccount `
                        -GcsNamespace $gcsNameSpace `
                        -GcsRegion $Using:gcsRegionName `
                        -GenevaConfigVersion $Using:genevaConfigVersion `
                        -LocalPath $cacheLocalPath `
                        -DisableUpdate "true" `
                        -MONITORING_AEO_REGION $Using:gcsRegionName `
                        -MONITORING_AEO_DEVICE_ARM_RESOURCE_URI $Using:deviceArmResourceUri `
                        -MONITORING_AEO_STAMPID $Using:stampId `
                        -MONITORING_AEO_CLUSTER_NAME $Using:clusterName `
                        -MONITORING_AEO_OSBUILD $Using:osBuild `
                        -MONITORING_AEO_ASSEMBLYBUILD $Using:assemblyVersion `
                        -MONITORING_AEO_NODEID $nodeId `
                        -MONITORING_AEO_NODE_ARC_RESOURCE_URI $arcAgentResourceId `
                        -MONITORING_AEO_CLUSTER_NODE_NAME "%COMPUTERNAME%"
                }
                $hostname = hostname
                Write-ObservabilityLog "SetGMATenantJsonRegistryKeys on $hostname succeeded."
            }
    }

    static GenerateTelemetryFromCachedFiles([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1

        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

        Write-ObservabilityLog "$env:COMPUTERNAME Generate Telemetry from Cached files start."

        $cloudRole      = $Parameters.Roles["Cloud"].PublicConfiguration
        $domainCredential = [ObservabilityConfig]::GetDomainCredential($Parameters)
        $cloudId = $cloudRole.PublicInfo.CloudId

        # Get host names using execution context
        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }

        $scriptBlock = {
            param (
                [string]
                [parameter(Mandatory=$true)]
                $CorrelationId

            )

            $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
            $logOrchestratorNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.LogOrchestrator"
            Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"

            $result = Invoke-CachedTelemetryFilesParsing `
                        -LogOrchestratorNugetPath $logOrchestratorNugetPath `
                        -CorrelationId $CorrelationId

            return $result
        }

        # Install Arc for server agent on each Host
        $exceptionMessage = ""
        foreach($hostName in $hostNames)
        {
            $argList = @($cloudId)
            $result = Invoke-Command `
                -ComputerName $hostName `
                -ScriptBlock $scriptBlock `
                -ArgumentList $argList `
                -Credential $domainCredential `
                -Authentication Credssp
            if ($result -eq $true)
            {
                Write-ObservabilityLog "Generate Telemetry from Cached files on host $hostName succeeded."
            }
            else
            {
                Write-ObservabilityLog "Generate Telemetry from Cached files on host $hostName failed with exception $result."
                $exceptionMessage += "${hostName}: $result`n"
            }
        }

        if($exceptionMessage)
        {
            throw $exceptionMessage
        }

        Write-ObservabilityLog "$env:COMPUTERNAME Generate Telemetry from Cached files end."
    }

    static [Hashtable] GetRegistrationParameters([CloudEngine.Configurations.EceInterfaceParameters] $Parameters)
    {
        Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1
        $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment"
        Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1"
        $cloudDeploymentNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.CloudDeployment"
        Import-Module Az.Accounts -Force

        $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters
        if($null -eq $hostNames)
        {
            $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        }


        Trace-Execution "Getting registration parameters from ECE role parameters"
        $nodeNames      = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name
        $cloudRole      = $Parameters.Roles["Cloud"].PublicConfiguration
        $securityInfo   = $cloudRole.PublicInfo.SecurityInfo
        $registrationParams = @{
            EnvironmentName         = $cloudRole.PublicInfo.RegistrationCloudName
            SubscriptionId          = $cloudRole.PublicInfo.RegistrationSubscriptionId
            TenantId                = $cloudRole.PublicInfo.RegistrationTenantId
            Region                  = $cloudRole.PublicInfo.RegistrationRegion
            ResourceGroupName       = $cloudRole.PublicInfo.RegistrationResourceGroupName
            ResourceName            = $cloudRole.PublicInfo.RegistrationResourceName
            CloudId                 = $cloudRole.PublicInfo.CloudId
            EnableAzureArcServer    = [System.Convert]::ToBoolean($cloudRole.PublicInfo.EnableAzureArcServer)
            ComputerName            = $nodeNames | Select-Object -First 1
        }

        if (![String]::IsNullOrEmpty($cloudRole.PublicInfo.RegistrationArcServerResourceGroupName)) {
            Trace-Execution "Using RegistrationArcServerResourceGroupName $($cloudRole.PublicInfo.RegistrationArcServerResourceGroupName) for ArcServer registration"
            $registrationParams.ResourceGroupName = $cloudRole.PublicInfo.RegistrationArcServerResourceGroupName
        }

        Trace-Execution "Getting AccessTokenfor ArcForServer using $($registrationParams | ConvertTo-Json -Depth 2)"
        $registrationParameterSet = $cloudRole.PublicInfo.RegistrationParameterSet
        if ($registrationParameterSet -eq "DefaultSet") {
            Trace-Execution "RegistrationParameterSet $registrationParameterSet, getting access tokens using user token cache"
            $registrationTokenCacheUser = $securityInfo.AADUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.RegistrationTokenCacheID
            $registrationTokenCacheCred = $Parameters.GetCredential($registrationTokenCacheUser.Credential)
            Trace-Execution "RegistrationTokenCacheCred $($registrationTokenCacheCred.UserName)"
            $clientId = $cloudRole.PublicInfo.RegistrationClientId
            Trace-Execution "Using clientId $clientId to get access token"
            Trace-Execution "Going to use RegistrationHelpers $cloudDeploymentNugetPath\content\Setup\Common\RegistrationHelpers.psm1"
            Import-Module "$cloudDeploymentNugetPath\content\Setup\Common\RegistrationHelpers.psm1"
            $armAccessToken     = Get-AccessToken -AzureEnvironment $registrationParams.EnvironmentName -TenantId $registrationParams.TenantId -TokenCacheCred $registrationTokenCacheCred -ClientId $clientId
            Trace-Execution "Access tokens using token cache is $($armAccessToken.AccessToken.Length)"

            $registrationParams += @{
                AccountId           = $registrationTokenCacheCred.UserName
                ArmAccessToken      = $armAccessToken.AccessToken
            }
        } else {
            Trace-Execution "RegistrationParameterSet $registrationParameterSet, getting access tokens using service principal"
            $registrationSPUser = $securityInfo.AADUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.RegistrationSPID
            $registrationSPCred = $Parameters.GetCredential($registrationSPUser.Credential)
            Trace-Execution "RegistrationSPCred AppId $($registrationSPCred.UserName)"
            Login-AzAccount -Environment $registrationParams.EnvironmentName -Credential $registrationSPCred -Tenant $registrationParams.TenantId -ServicePrincipal
            $armAccessToken = Get-AzAccessToken -Verbose
            Trace-Execution "Access tokens using service principal is $($armAccessToken.Token.Length)"

            $registrationParams += @{
                AccountId           = $registrationSPCred.UserName
                ArmAccessToken      = $armAccessToken.Token
            }
        }

        if($registrationParams.ArmAccessToken.Length -eq 0)
        {
            throw "GetRegistrationParameters failed to retrieve AccessToken"
        }

        return $registrationParams
    }
}
# SIG # Begin signature block
# MIIoKgYJKoZIhvcNAQcCoIIoGzCCKBcCAQExDzANBglghkgBZQMEAgEFADB5Bgor
# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAdj1hii+gxYFcO
# dCUnjeyfCreFdk9KIeDvmn+pPIpP+aCCDXYwggX0MIID3KADAgECAhMzAAADTrU8
# esGEb+srAAAAAANOMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p
# bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI5WhcNMjQwMzE0MTg0MzI5WjB0MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
# AQDdCKiNI6IBFWuvJUmf6WdOJqZmIwYs5G7AJD5UbcL6tsC+EBPDbr36pFGo1bsU
# p53nRyFYnncoMg8FK0d8jLlw0lgexDDr7gicf2zOBFWqfv/nSLwzJFNP5W03DF/1
# 1oZ12rSFqGlm+O46cRjTDFBpMRCZZGddZlRBjivby0eI1VgTD1TvAdfBYQe82fhm
# WQkYR/lWmAK+vW/1+bO7jHaxXTNCxLIBW07F8PBjUcwFxxyfbe2mHB4h1L4U0Ofa
# +HX/aREQ7SqYZz59sXM2ySOfvYyIjnqSO80NGBaz5DvzIG88J0+BNhOu2jl6Dfcq
# jYQs1H/PMSQIK6E7lXDXSpXzAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE
# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUnMc7Zn/ukKBsBiWkwdNfsN5pdwAw
# RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW
# MBQGA1UEBRMNMjMwMDEyKzUwMDUxNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci
# tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG
# CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu
# Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0
# MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAD21v9pHoLdBSNlFAjmk
# mx4XxOZAPsVxxXbDyQv1+kGDe9XpgBnT1lXnx7JDpFMKBwAyIwdInmvhK9pGBa31
# TyeL3p7R2s0L8SABPPRJHAEk4NHpBXxHjm4TKjezAbSqqbgsy10Y7KApy+9UrKa2
# kGmsuASsk95PVm5vem7OmTs42vm0BJUU+JPQLg8Y/sdj3TtSfLYYZAaJwTAIgi7d
# hzn5hatLo7Dhz+4T+MrFd+6LUa2U3zr97QwzDthx+RP9/RZnur4inzSQsG5DCVIM
# pA1l2NWEA3KAca0tI2l6hQNYsaKL1kefdfHCrPxEry8onJjyGGv9YKoLv6AOO7Oh
# JEmbQlz/xksYG2N/JSOJ+QqYpGTEuYFYVWain7He6jgb41JbpOGKDdE/b+V2q/gX
# UgFe2gdwTpCDsvh8SMRoq1/BNXcr7iTAU38Vgr83iVtPYmFhZOVM0ULp/kKTVoir
# IpP2KCxT4OekOctt8grYnhJ16QMjmMv5o53hjNFXOxigkQWYzUO+6w50g0FAeFa8
# 5ugCCB6lXEk21FFB1FdIHpjSQf+LP/W2OV/HfhC3uTPgKbRtXo83TZYEudooyZ/A
# Vu08sibZ3MkGOJORLERNwKm2G7oqdOv4Qj8Z0JrGgMzj46NFKAxkLSpE5oHQYP1H
# tPx1lPfD7iNSbJsP6LiUHXH1MIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq
# hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x
# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv
# bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
# IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG
# EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG
# A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg
# Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
# CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03
# a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr
# rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg
# OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy
# 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9
# sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh
# dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k
# A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB
# w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn
# Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90
# lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w
# ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o
# ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD
# VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa
# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny
# bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG
# AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t
# L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV
# HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3
# dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG
# AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl
# AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb
# C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l
# hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6
# I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0
# wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560
# STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam
# ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa
# J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah
# XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA
# 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt
# Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr
# /Xmfwb1tbWrJUnMTDXpQzTGCGgowghoGAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw
# EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN
# aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp
# Z25pbmcgUENBIDIwMTECEzMAAANOtTx6wYRv6ysAAAAAA04wDQYJYIZIAWUDBAIB
# BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO
# MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIHoPAB+j2qRUtc6egmQS9xxI
# +tXsxZhmCB0CIrjkSYGYMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A
# cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB
# BQAEggEAyALfvnmGgbGqT6fBudiafQ6h03tfpwIqfaXUO1+l3DbrtBh2POXmgBSg
# BMXwOxBdMhHeMxOA22LSCEuhfpmwyFv4sySKBymjMGFmYsz4aIHl0Ot3yJbg8Qom
# hsdFow360Wf9P9ZUNZ1pM72qjI4yseonD/iML8Pc0+axhZdL3stRF40X1NcEOWVA
# wwsghh0QMbWhopW+ZU+20MKpnnPHz7XiqliKKgL7oF9DT4aLZzoxGbF8t9Bv1wDH
# /jVMHSG6J3YqB7GVh5LINO/DyvxhrmvOMgjjNE5+dRtxdsdvznbzciV06/4UGibv
# g0TT30nuzd2rnQujlp91q6Sj26l8r6GCF5QwgheQBgorBgEEAYI3AwMBMYIXgDCC
# F3wGCSqGSIb3DQEHAqCCF20wghdpAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFSBgsq
# hkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl
# AwQCAQUABCDH7/D0cZOtbAV6Bc6tAWgqhWZICvt+5dE0wYYMWhSjaAIGZMvWr8kO
# GBMyMDIzMDgwNzIxMzMzMC4xMjdaMASAAgH0oIHRpIHOMIHLMQswCQYDVQQGEwJV
# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE
# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1l
# cmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046ODYwMy0w
# NUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2Wg
# ghHqMIIHIDCCBQigAwIBAgITMwAAAdebDR5XLoxRjgABAAAB1zANBgkqhkiG9w0B
# AQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE
# BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD
# VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAeFw0yMzA1MjUxOTEy
# MzdaFw0yNDAyMDExOTEyMzdaMIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz
# aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv
# cnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25z
# MScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046ODYwMy0wNUUwLUQ5NDcxJTAjBgNV
# BAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggIiMA0GCSqGSIb3DQEB
# AQUAA4ICDwAwggIKAoICAQDErGCkN2X/UvuNCcfl0yVBNo+LIIyzG7A10X5kVgGn
# p9s8mf4aZsukZu5rvLs7NqaNExcwnPuHIWdp6kswja1Yw9SxTX+E0leq+WBucIRK
# WdcMumIDBgLE0Eb/3/BY95ZtT1XsnnatBFZhr0uLkDiT9HgrRb122sm7/YkyMigF
# kT0JuoiSPXoLL7waUE9teI9QOkojqjRlcIC4YVNY+2UIBM5QorKNaOdz/so+TIF6
# mzxX5ny2U/o/iMFVTfvwm4T8g/Yqxwye+lOma9KK98v6vwe/ii72TMTVWwKXFdXO
# ysP9GiocXt38cuP9c8aE1eH3q4FdGTgKOd0rG+xhCgsRF8GqLT7k58VpQnJ8u+yj
# RW6Lomt5Rcropgf9EH8e4foDUoUyU5Q7iPgwOJxYhoKxRjGZlthDmp5ex+6U6zv9
# 5rd973668pGpCku0IB43L/BTzMcDAV4/xu6RfcVFwarN/yJq5qfZyMspH5gcaTCV
# AouXkQTc8LwtfxtgIz53qMSVR9c9gkSnxM5c1tHgiMX3D2GBnQan95ty+CdTYAAh
# jgBTcyj9P7OGEMhr3lyaZxjr3gps6Zmo47VOTI8tsSYHhHtD8BpBog39L5e4/lDJ
# g/Oq4rGsFKSxMXuIRZ1E08dmX67XM7qmvm27O804ChEmb+COR8Wb46MFEEz62ju+
# xQIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFK6nwLv9WQL3NIxEJyPuJMZ6MI2NMB8G
# A1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8GA1UdHwRYMFYwVKBSoFCG
# Tmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUy
# MFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBsBggrBgEFBQcBAQRgMF4w
# XAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2Vy
# dHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3J0MAwG
# A1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwgwDgYDVR0PAQH/BAQD
# AgeAMA0GCSqGSIb3DQEBCwUAA4ICAQBSBd3UJ+IsvdMCX+K7xqHa5UBtVC1CaXZv
# HRd+stW0lXA/dTNneCW0TFrBoJY59b9fnbTouPReaku2l3X5bmhsao6DCRVuqcmh
# VPAZySXGeoVfj52cLGiyZLEw6TQzu6D++vjNOGmSibO0KE9Gdv8hQERx5RG0KgrT
# mk8ckeC1VUqueUQHKVCESqTDUDD8dXTLWCmm6HqmQX6/+gKDSXggwpc75hi2AbKS
# o4tulMwTfXJdGdwrsiHjkz8nzIW/Z3PnMgGFU76KuzYFV0XyH9DTS/DPO86RLtQj
# A5ZlVGymTPfTnw7kxoiLJN/yluMHIkHSzpaJvCiqX+Dn1QGREEnNIZeRekvLourq
# PREIOTm1bJRJ065c9YX7bJ0naPixzm5y8Y2B+YIIEAi4jUraOh3oE7a4JvIW3Eg3
# oNqP7qhpd7xMLxq2WnM+U9bqWTeT4VCopAhXu2uGQexdLq7bWdcYwyEFDhS4Z9N0
# uw3h6bjB7S4MX96pfYSEV0MKFGOKbmfCUS7WemkuFqZy0oNHPPx+cfdNYeSF6bhO
# PHdsro1EVd3zWIkdD1G5kEDPnEQtFartM8H+bv5zUhAUJs8qLzuFAdBZQLueD9XZ
# eynjQKwEeAz63xATICh8tOUM2zMgSEhVL8Hm45SB6foes4BTC0Y8SZWov3Iahtvw
# yHFbUqs1YjCCB3EwggVZoAMCAQICEzMAAAAVxedrngKbSZkAAAAAABUwDQYJKoZI
# hvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw
# DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x
# MjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAy
# MDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIyNVowfDELMAkGA1UEBhMC
# VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV
# BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp
# bWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC
# AQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXIyjVX9gF/bErg4r25Phdg
# M/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjoYH1qUoNEt6aORmsHFPPF
# dvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1yaa8dq6z2Nr41JmTamDu6
# GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v3byNpOORj7I5LFGc6XBp
# Dco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pGve2krnopN6zL64NF50Zu
# yjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viSkR4dPf0gz3N9QZpGdc3E
# XzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYrbqgSUei/BQOj0XOmTTd0
# lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlMjgK8QmguEOqEUUbi0b1q
# GFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSLW6CmgyFdXzB0kZSU2LlQ
# +QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AFemzFER1y7435UsSFF5PA
# PBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIurQIDAQABo4IB3TCCAdkw
# EgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIEFgQUKqdS/mTEmr6CkTxG
# NSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMFwGA1UdIARV
# MFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWlj
# cm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0bTATBgNVHSUEDDAK
# BggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMC
# AYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvX
# zpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20v
# cGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYI
# KwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNydDANBgkqhkiG
# 9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv6lwUtj5OR2R4sQaTlz0x
# M7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZnOlNN3Zi6th542DYunKmC
# VgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1bSNU5HhTdSRXud2f8449
# xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4rPf5KYnDvBewVIVCs/wM
# nosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU6ZGyqVvfSaN0DLzskYDS
# PeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDFNLB62FD+CljdQDzHVG2d
# Y3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/HltEAY5aGZFrDZ+kKNxn
# GSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdUCbFpAUR+fKFhbHP+Crvs
# QWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKiexcdFYmNcP7ntdAoGokL
# jzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTmdHRbatGePu1+oDEzfbzL
# 6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZqELQdVTNYs6FwZvKhggNN
# MIICNQIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp
# bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw
# b3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2EgT3BlcmF0aW9uczEn
# MCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjg2MDMtMDVFMC1EOTQ3MSUwIwYDVQQD
# ExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMKAQEwBwYFKw4DAhoDFQAx
# W9uizG3hEY89uL2uu+X+mG/rdaCBgzCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w
# IFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA6Hua9TAiGA8yMDIzMDgwNzE2MzE0
# OVoYDzIwMjMwODA4MTYzMTQ5WjB0MDoGCisGAQQBhFkKBAExLDAqMAoCBQDoe5r1
# AgEAMAcCAQACAipRMAcCAQACAhOPMAoCBQDofOx1AgEAMDYGCisGAQQBhFkKBAIx
# KDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEAAgMBhqAwDQYJKoZI
# hvcNAQELBQADggEBABdn3m52hFv/2hlQzkyvyVPoHSOjzA+h7lj82KzEax8eBbhJ
# TzxCJgaV36q3DE9SctjYH43XajSYHz3KGTdlGN5ljuTpEW3lGTItrEDGg8pfg9uZ
# cQcd6pG3QuF5lIjpoIPc8qPuBfniH08MrjeIrx5lp9netqSLfCqcE+ualK6hG66f
# PzXnHrAWtu2Yf6uzUQYGSnXkG3zn0JFUt8nsaE6mg2xBAV3KAHV9BrZVLRjOFOwk
# Kr2FMzDz+4UKUuHe35/yBhARXRugyxGhkyyYsYp/x4eot/LJZ5JF2FP2XoA5sjLd
# dHHfO/pl8YMV1LUVn5Top+rG7H6jDI909ruqwugxggQNMIIECQIBATCBkzB8MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNy
# b3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAdebDR5XLoxRjgABAAAB1zAN
# BglghkgBZQMEAgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8G
# CSqGSIb3DQEJBDEiBCAzvkUDOFKlAzS8nwtuQsx9BfpyZg7DkH5mO2GAiKfUwTCB
# +gYLKoZIhvcNAQkQAi8xgeowgecwgeQwgb0EIJzePl5LXn1PiqNjx8YN7TN1ZI0d
# 1ZX/2zRdnI97rJo7MIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldh
# c2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBD
# b3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIw
# MTACEzMAAAHXmw0eVy6MUY4AAQAAAdcwIgQg7twZ4hxbMymMSddZv9lcCJJ0/6ZZ
# tdZbXnUogFmX5C0wDQYJKoZIhvcNAQELBQAEggIAdlIJt7eQLP7P07rV53ftFxnW
# yL6EBlgt3obRUU0r7xa3rN+/h5D9GCFaaIuy5bSZOLiSEKKoMqvMYMVjxxZAEcvC
# qWy7oLuaJbMgd4ESMKmg6GxOKmjsQmznsQLu0UibwKI8XGH7ACbH4DeXxaiDSO8g
# RhyAJMWudZysFBVPVo/ucn6/O5RDdlXg9JiyUFQawSjClCBpr2ikHZKo6z2vtwsJ
# VJxnA1FPEeTqAUI3sq4R+acP+iOkOH/pyZyOQlrDJA8sRUttYPeX5y9BiYwvs2CN
# 6nWfCLVk9i4XxEsBhScWV+eXNjWw4l3nYbYicTUi47XUmqFXXiWsdQQKmgd7ZWsn
# w8jv5P9MKFNAPS8jDFvwhdNm3PIWtZKZxMFV685f3aB31TMqCZAuXWU6hlvDVnUD
# +3Q4HV+js2Ekbk0pTP9NiWsJ1Yxyw3H2rC3NWrO5lYfBn851QAxfknSf9j2a1tp+
# 8TGuOqW3T0z424/g9S252psYCWkvLD8/4ApkzZNj136QSsELntSo9zQ3bbHh59Qp
# pJq6E/AzKVNnnNy9xrvpq67UbZuGk4yjYf+HwKRihmnipKFgXTEF/KdSsTfZNGCa
# fPdGI5OR3gV2F93CYM86fwNCj4wUzrDZmczthcPXZdwMLJP9YTjEl70vqRCQ2Ai5
# b//p7d+fVkU2cwvxp78=
# SIG # End signature block