Obs/bin/ObsDep/content/Powershell/ObservabilityHelpers.psm1
<###################################################
# # # Copyright (c) Microsoft. All rights reserved. # # # ##################################################> # Import Observability EventSource $observabilityNugetPath = "$PSScriptRoot\..\.." Add-Type -Path "$observabilityNugetPath\lib\net472\Microsoft.AzureStack.Observability.ObservabilityDeployment.dll" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityConstants.psm1" # Create and setup new Observability volume. Returns tuple with computer name and $true on success, or error information $_ on failure. function Add-ObservabilityVolumeWithRetry { [CmdletBinding(SupportsShouldProcess=$true, ConfirmImpact='Medium')] param ( # Fully qualified path to the VHD file that should be created. # An exception is thwrown if that VHD file already exist. This approach is chosen # to prevent issues, should the VHD file be still mounted. [string] [Parameter(Mandatory=$true)] $Path, # Folder access path for mounted drive. [string] [Parameter(Mandatory=$true)] $AccessPath, # Volume name for the partition that is created within the VHD [string] [Parameter(Mandatory=$true)] $VolumeLabel, # Maximum size in bytes for the VHD file. [int64] [Parameter(Mandatory=$true)] $Size, # Optional. Specifies that the VHD should be static. [switch] [Parameter(Mandatory=$false)] $StaticSize, # Optional. Number of times to retry operation before failing. [int] [Parameter(Mandatory=$false)] $Retries = 5, # Optional. Seconds to sleep before retry [int] [Parameter(Mandatory=$false)] $RetrySleepTimeInSeconds = 10 ) if(Test-Path $Path) { if((Get-DiskImage $Path).Attached) { Write-ObservabilityLog "VHD file is already mounted." } else { Write-ObservabilityLog "Existing unmounted VHD file found. Mounting $Path" Mount-VHD -Path $Path -NoDriveLetter -Verbose } } else { $retryAttempt = 0 $success = $false $hostName = $env:COMPUTERNAME while(-not($success) -and ($retryAttempt -lt $Retries)) { $retryAttempt = $retryAttempt + 1 try { Write-ObservabilityLog "Trying to create Observability Volume. Attempt $retryAttempt of $Retries" if($StaticSize) { New-ObservabilityVolume -Path $Path -AccessPath $AccessPath -VolumeLabel $VolumeLabel -Size $Size -StaticSize } else { New-ObservabilityVolume -Path $Path -AccessPath $AccessPath -VolumeLabel $VolumeLabel -Size $Size } Write-ObservabilityVolumeCreationStopTelemetry ` -ComputerName $hostName ` -Message "Observability volume setup on host $hostName succeeded." $success = $true } catch { $exceptionMessage = $_.Exception.Message Write-ObservabilityVolumeCreationStopTelemetry ` -ComputerName $hostName ` -Message "Observability volume setup on host $hostName failed with exception $exceptionMessage." ` -ExceptionDetails $exceptionMessage if ($retryAttempt -lt $Retries) { Write-ObservabilityErrorLog "Failure during VHD creation: '$exceptionMessage'. Retrying." } else { # All retries failed. throw $_ } Start-Sleep -Seconds $RetrySleepTimeInSeconds } } } } # Function to create a data VHD file, which contains an empty, NTFS formatted partition. Returns $true on success, $false on failure function New-ObservabilityVolume { [CmdletBinding(SupportsShouldProcess=$true, ConfirmImpact='Medium')] param ( # Fully qualified path to the VHD file that should be created. # An exception is thwrown if that VHD file already exist. This approach is chosen # to prevent issues, should the VHD file be still mounted. [string] [Parameter(Mandatory=$true)] $Path, # Folder access path for mounted drive. [string] [Parameter(Mandatory=$true)] $AccessPath, # Volume name for the partition that is created within the VHD. [string] [Parameter(Mandatory=$true)] $VolumeLabel, # Maximum size in bytes for the VHD file. [int64] [Parameter(Mandatory=$true)] $Size, # Optional. Specifies that the VHD should be static. [switch] [Parameter(Mandatory=$false)] $StaticSize ) Set-StrictMode -Version Latest $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop Import-Module Hyper-V $disk = $null try { Write-ObservabilityLog ("Creating new VHD $Path with the following properties: Max size: $Size bytes, Static: $StaticSize.") if ($StaticSize) { New-VHD -Path $Path -SizeBytes $Size -Fixed -Verbose | Out-Null } else { New-VHD -Path $Path -SizeBytes $Size -Dynamic -Verbose | Out-Null } } catch { Remove-ObservabilityVolume -Path $Path -AccessPath $AccessPath throw ("Error creating the data VHD file at {0}. Error: {1}" -f $Path, $_) } try { Write-ObservabilityLog ("Mounting a data VHD {0}" -f $Path) $disk = Mount-VHD -Path $Path -Passthru -NoDriveLetter -Verbose Write-ObservabilityLog "Initializing mounted VHD for disk number $($disk.Number)" $null = Initialize-Disk -Number $disk.Number -Verbose Write-ObservabilityLog "Creating new partition. for disk number $($disk.Number)" $part = New-Partition -DiskNumber $disk.Number -UseMaximumSize -Verbose Write-ObservabilityLog "Formatting volume." $null = Format-Volume -Partition $part -FileSystem NTFS -NewFileSystemLabel $VolumeLabel -Force -Confirm:$false -Verbose Write-ObservabilityLog ("Setting attribute '{0}' to '{1}'." -f "NoDefaultDriveLetter", "true") Set-Partition $part.DiskNumber $part.PartitionNumber -NoDefaultDriveLetter $true -Verbose $diskString = Out-String -InputObject $disk Write-ObservabilityLog $diskString } catch { Remove-ObservabilityVolume -Path $Path -AccessPath $AccessPath throw New-Object -TypeName System.ApplicationException -ArgumentList ($("Mounting VHD and formatting the partition failed. Error: {0}" -f $_.Exception.Message), $_.Exception) } } # Dismount and Delete vhdx file at given path function Remove-ObservabilityVolume { param( [string] [Parameter(Mandatory=$true)] $Path, [string] [Parameter(Mandatory=$true)] $AccessPath ) if(Test-Path $Path) { try { if((Get-DiskImage $Path).Attached) { Dismount-VHD -Path $Path -Verbose } } catch {} $null = Remove-Item -Path $Path -Force } if(Test-Path $AccessPath) { $null = Remove-Item -Path $AccessPath -Force } } # Tests if path is an empty directory(including hidden files) Note: Will return false if path is not a directory function Test-DirectoryIsEmpty { param( [string] [Parameter(Mandatory=$true)] $Path ) return (Get-ChildItem $Path -Force).Count -eq 0 } # Create and set quotas for each subfolder of the Observability volume. Returns tuple with computer name and $true on success, or error information $_ on failure. function New-VolumeFoldersAndPrunerWithRetry { param ( # Folder access path for mounted drive. [string] [parameter(Mandatory=$true)] $AccessPath, # For folders with enforced quota, when the folder is this percent full cleanup will be initiated. [int] [Parameter(Mandatory=$true)] $CleanupThresholdPercent, # For folders with enforced quota, then cleanup occurs files will be pruned until this fullness percentage is reached. [int] [Parameter(Mandatory=$true)] $FreeSpaceThresholdPercent, # Frequency for which each folder pruning scheduled task executes [int] [Parameter(Mandatory=$true)] $PurgeFolderFrequencyInMinutes, # Name of subfolder config file to read [string] [Parameter(Mandatory=$true)] $SubFolderConfigFileName, # Optional. Number of times to retry operation before failing. [int] [Parameter(Mandatory=$false)] $Retries = 5, # Optional. Seconds to sleep before retry [int] [Parameter(Mandatory=$false)] $RetrySleepTimeInSeconds = 10 ) $retryAttempt = 0 $success = $false while(-not($success) -and ($retryAttempt -lt $Retries)) { $retryAttempt = $retryAttempt + 1 try { Write-ObservabilityLog "Trying to setup Observability Folder quotas. Attempt $retryAttempt of $Retries" New-VolumeFoldersAndPruner ` -AccessPath $AccessPath ` -CleanupThresholdPercent $CleanupThresholdPercent ` -FreeSpaceThresholdPercent $FreeSpaceThresholdPercent ` -PurgeFolderFrequencyInMinutes $PurgeFolderFrequencyInMinutes ` -SubFolderConfigFileName $SubFolderConfigFileName Write-ObservabilityLog "Observability volume folder and pruner setup on host $env:COMPUTERNAME succeeded." $success = $true } catch { if ($retryAttempt -lt $Retries) { $exceptionMessage = $_.Exception.Message Write-ObservabilityErrorLog "Failure during quota setup: '$exceptionMessage'. Retrying." } else { # All retries failed. throw $_ } Start-Sleep -Seconds $RetrySleepTimeInSeconds } } } # Create each subfolder of observability volume and set folder quota if specified. function New-VolumeFoldersAndPruner { param ( # Folder access path for mounted drive. [string] [parameter(Mandatory=$true)] $AccessPath, # When a folder is this percent full cleanup will be initiated. [int] [Parameter(Mandatory=$true)] $CleanupThresholdPercent, # For folders with enforced quota, then cleanup occurs files will be pruned until this fullness percentage is reached. [int] [Parameter(Mandatory=$true)] $FreeSpaceThresholdPercent, # Frequency for which each folder cleanup scheduled task executes [int] [Parameter(Mandatory=$true)] $PurgeFolderFrequencyInMinutes, # Name of subfolder config file to read [string] [Parameter(Mandatory=$true)] $SubFolderConfigFileName ) $observabilityNugetPath = "$PSScriptRoot\..\.." $pruneObservabilityPath = "$observabilityNugetPath\content\Powershell\PruneObservabilityVolume.ps1" $subFolders = Get-ObservabilityFolders -SubFolderConfigFileName $SubFolderConfigFileName foreach ($subFolder in $subFolders) { $subFolderPath = Join-Path -Path $AccessPath -ChildPath $subFolder.Name if(-not (Test-Path $subFolderPath)){ Write-ObservabilityLog ("Creating empty directory at $subFolderPath.") New-Item -Path $subFolderPath -ItemType "Directory" | Out-Null } } # Create Scheduled task to prune observability volume $taskName = "ObservabilityVolumePruner" if(Test-ScheduledTaskExists -TaskName $taskName) { Unregister-ScheduledTask -TaskName $taskName -Confirm:$false | Out-Null } Write-ObservabilityLog "Creating new scheduled task $taskName." $frequency = New-TimeSpan -Minutes $PurgeFolderFrequencyInMinutes $principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount $trigger = New-ScheduledTaskTrigger -RepetitionInterval $frequency -Once -At "12:00 AM" $action = New-ScheduledTaskAction ` -Execute "powershell.exe" ` -Argument "-Command $pruneObservabilityPath -AccessPath $AccessPath -CleanupThresholdPercent $CleanupThresholdPercent -FreeSpaceThresholdPercent $FreeSpaceThresholdPercent -SubFolderConfigFileName $SubFolderConfigFileName" $settings = New-ScheduledTaskSettingsSet $task = New-ScheduledTask -Action $action -Trigger $trigger -Settings $settings -Principal $principal Register-ScheduledTask -TaskName $taskName -TaskPath "Microsoft\AzureStack\Observability" -InputObject $task Write-ObservabilityLog "Creating new scheduled task $taskName succeeded." Add-VolumeFolderQuotaStatisticsScheduledTask -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName } function Set-FolderQuotas { param ( # Folder access path for mounted drive. [string] [parameter(Mandatory=$true)] $AccessPath, # Name of subfolder config file to read [string] [Parameter(Mandatory=$true)] $SubFolderConfigFileName ) $subFolders = Get-ObservabilityFolders -SubFolderConfigFileName $SubFolderConfigFileName foreach($subFolder in $subFolders) { $subFolderPath = Join-Path -Path $AccessPath -ChildPath $subFolder.Name if(Test-Path $subFolderPath){ if(Test-PathHasQuota -Path $subFolderPath) { Remove-FsrmQuota -Path $subFolderPath -Confirm:$false -Verbose | Out-Null } } else { Write-ObservabilityLog ("Creating empty directory at $subFolderPath.") New-Item -Path $subFolderPath -ItemType "Directory" -Verbose | Out-Null } $subFolderSizeInMB = [int64]::Parse($subFolder.SizeInMB) $subFolderSizeInBytes = $subFolderSizeInMB * 1MB Write-ObservabilityLog "Creating new FSRM quota at path $subFolderPath of size $subFolderSizeInBytes bytes." New-FsrmQuota -Path $subFolderPath -Size $subFolderSizeInBytes -ErrorAction Stop -Verbose | Out-Null Write-ObservabilityLog "Creating FSRM quota at path $subFolderPath succeeded." } } # Returns true if a given task exists and false otherwise. function Test-ScheduledTaskExists { param( # Absolute path of folder to test [string] [Parameter(Mandatory=$true)] $TaskName ) try { if(Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop) { return $true } } catch {} return $false } # Returns true if a given folder has a quota and false otherwise. function Test-PathHasQuota { param( # Absolute path of folder to test [string] [Parameter(Mandatory=$true)] $Path ) $quotas = Get-FsrmQuota -ErrorAction Stop -Verbose if ($quotas -is [System.Array]) { foreach ($quota in $quotas) { if ($quota.Path -eq $Path) { return $true } } return $false } else { return $quotas.Path -eq $Path } } # Remove oldest files from folder until free space threshold is reached function Remove-ObservabilityFolderOldestFiles { param( # Folder access path for mounted drive. [string] [parameter(Mandatory=$true)] $AccessPath, # When a folder is this percent full cleanup will be initiated. [int] [Parameter(Mandatory=$true)] $CleanupThresholdPercent, # When cleanup occurs files will be pruned until this fullness percentage is reached. [int] [Parameter(Mandatory=$true)] $FreeSpaceThresholdPercent, # Name of subfolder config file to read [string] [Parameter(Mandatory=$true)] $SubFolderConfigFileName ) $subFolders = Get-ObservabilityFolders -SubFolderConfigFileName $SubFolderConfigFileName foreach($subFolder in $subFolders) { $subFolderPath = Join-Path -Path $AccessPath -ChildPath $subFolder.Name $subFolderSizeInMB = [int64]::Parse($subFolder.SizeInMB) $quotaSize = $subFolderSizeInMB * 1MB $files = Get-ChildItem -Path $subFolderPath -Recurse -Attributes "!Directory" | Sort-Object -Property "LastWriteTime" $directorySize = ($files | Measure-Object -Property "Length" -Sum).Sum $freeSpaceSize = $quotaSize * $FreeSpaceThresholdPercent / 100 $cleanupThresholdSize = $quotaSize * $CleanupThresholdPercent / 100 if($directorySize -gt $cleanupThresholdSize) { Write-ObservabilityLog "Cleanup Trigger Threshold Percent of $CleanupThresholdPercent reached." Write-ObservabilityLog "Cleanup of oldest files from $subFolderPath started." foreach ($file in $files) { $filePath = $file.FullName $fileSize = $file.Length $exceptionDetails = "" $success = $false try { Write-ObservabilityLog "Removing file at $filePath of size $fileSize bytes." Remove-Item -Path $filePath -Force $directorySize = $directorySize - $fileSize $success = $true } catch { # Next activation of scheduled task will retry delete. $exceptionDetails = $_ $success = $false } finally { Write-ObservabilityVolumePrunerFileDeletionTelemetry ` -ComputerName $ENV:COMPUTERNAME ` -DirectoryPath $subFolderPath ` -DirectorySize $directorySize ` -FilePath $filePath ` -FileSize $fileSize ` -Success $success ` -ExceptionDetails $exceptionDetails } if ($directorySize -lt $freeSpaceSize) { Write-ObservabilityLog "Acceptable volume size of $freeSpaceSize reached. Stopping cleanup." break } } Write-ObservabilityLog "Cleanup of oldest files from $subFolderPath completed." } } } # Returns object array of the subfolders within the Observability volume. # Folders should include name and folder size as percentage of volume size. function Get-ObservabilityFolders { param( # Name of subfolder config file to read [string] [Parameter(Mandatory=$true)] $SubFolderConfigFileName ) $observabilityNugetPath = "$PSScriptRoot\..\.." $observabilityFoldersFilePath = "$observabilityNugetPath\content\Configuration\$SubFolderConfigFileName" return Get-Content $observabilityFoldersFilePath | ConvertFrom-Json } # Creates a scheduled task to mount the Observability volume on startup, and to attempt to do so every hour. function Add-MountVolumeScheduledTask { param ( # Fully qualified path to the VHD file that should be created. # An exception is thrown if that VHD file already exist. This approach is chosen # to prevent issues, should the VHD file be still mounted. [string] [Parameter(Mandatory=$true)] $Path, # Credential under which the scheduled task should run. If not supplied, the system account is used. [PSCredential] [Parameter(Mandatory=$false)] $Credential ) $taskName = "MountObservabilityVolume" if(Test-ScheduledTaskExists -TaskName $taskName) { Unregister-ScheduledTask -TaskName $taskName -Confirm:$false | Out-Null } # Setup scheduled task to mount VHD on startup $observabilityNugetPath = "$PSScriptRoot\..\.." $mountScriptPath = "$observabilityNugetPath\content\Powershell\MountObservabilityVolume.ps1" Write-ObservabilityLog "Creating new scheduled task $taskName." $frequency = New-TimeSpan -Minutes 30 $hourlyTrigger = New-ScheduledTaskTrigger -RepetitionInterval $frequency -Once -At "12:00 AM" $startupTrigger= New-ScheduledTaskTrigger -AtStartup $triggers = @($startupTrigger, $hourlyTrigger) $action = New-ScheduledTaskAction ` -Execute "powershell.exe" ` -Argument "-Command $mountScriptPath -Path $Path" $settings = New-ScheduledTaskSettingsSet if($Credential) { $principal = New-ScheduledTaskPrincipal -LogonType S4U -UserId $Credential.UserName $task = New-ScheduledTask -Action $action -Trigger $triggers -Settings $settings -Principal $principal Register-ScheduledTask ` -TaskName $taskName ` -TaskPath "Microsoft\AzureStack\Observability" ` -InputObject $task ` -User $Credential.UserName ` -Password $Credential.GetNetworkCredential().Password } else { $principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount $task = New-ScheduledTask -Action $action -Trigger $triggers -Settings $settings -Principal $principal Register-ScheduledTask -TaskName $taskName -TaskPath "Microsoft\AzureStack\Observability" -InputObject $task } Write-ObservabilityLog "Creating new scheduled task $taskName succeeded." } function Add-VolumeAccessPath { param( [string] [Parameter(Mandatory=$true)] $AccessPath, [string] [Parameter(Mandatory=$true)] $VolumePath ) Write-ObservabilityLog ("Adding access path at $AccessPath.") if(-not (Get-DiskImage $VolumePath).Attached) { Mount-VHD -Path $VolumePath -Verbose -ErrorAction Stop } $diskNumber = (Get-DiskImage $VolumePath).Number $part = Get-Partition -DiskNumber $diskNumber -PartitionNumber 2 if (Test-Path $AccessPath) { if (Test-AccessPathExists -AccessPath $AccessPath -VolumeFilePath $VolumePath) { Write-ObservabilityLog ("Access path at $AccessPath already exists. Noop.") } elseif(-Not(Test-DirectoryIsEmpty -Path $AccessPath)) { throw "$AccessPath is not an empty directory. It cannot be used as a folder mount point." } } else { Write-ObservabilityLog ("Creating empty directory at $AccessPath.") New-Item -Path $AccessPath -ItemType "Directory" -Verbose | Out-Null if (Test-AccessPathExists -AccessPath $AccessPath -VolumeFilePath $VolumePath) { Write-ObservabilityLog ("Access path at $AccessPath already exists. Removing access path $AccessPath from $VolumePath.") Remove-Partitionaccesspath -InputObject $part -AccessPath $AccessPath -Verbose | Out-null } Add-PartitionAccessPath -InputObject $part -AccessPath $AccessPath -Verbose | Out-Null Write-ObservabilityLog ("Successfully added access path at $AccessPath.") } } # For a given mounted volume, return true if it has the specified access path and false if not. function Test-AccessPathExists { param( [string] [Parameter(Mandatory=$true)] $AccessPath, [string] [Parameter(Mandatory=$true)] $VolumeFilePath ) $disk = Get-VHD $VolumeFilePath $part = Get-Partition -Disknumber $disk.Number -PartitionNumber 2 $matchedAccessPath = $part.AccessPaths | Where-Object {$_ -clike "$AccessPath*"} if($matchedAccessPath) { return $true } else { return $false } } function Write-ObservabilityLog { param( [string] [Parameter(Mandatory=$true)] $Message ) if (Get-Command Trace-Execution -ErrorAction SilentlyContinue) { Trace-Execution $Message } [Microsoft.AzureStack.Observability.ObservabilityEventSource]::Log.WriteInformational($Message) } function Write-ObservabilityErrorLog { param( [string] [Parameter(Mandatory=$true)] $Message ) if (Get-Command Trace-Execution -ErrorAction SilentlyContinue) { Trace-Execution $Message } [Microsoft.AzureStack.Observability.ObservabilityEventSource]::Log.WriteError($Message) } function Write-ObservabilityVolumeCreationStartTelemetry { param( [string] [Parameter(Mandatory=$true)] $ComputerNames, [string] [Parameter(Mandatory=$true)] $VolumeFilePath, [string] [Parameter(Mandatory=$true)] $VolumeAccessPath, [string] [Parameter(Mandatory=$true)] $VolumeLabel, [string] [Parameter(Mandatory=$true)] $VolumeSize ) [Microsoft.AzureStack.Observability.ObservabilityConfigTelemetryEventSource]::Log.ObservabilityVolumeCreationStart( $ComputerNames, $VolumeFilePath, $VolumeAccessPath, $VolumeLabel, $VolumeSize ) $volumeInfo = @" Starting Observability Volume Setup ComputerNames: $ComputerNames Volume path: $volumeFilePath Volume folder mount access path: $VolumeAccessPath Volume label: $VolumeLabel Volume size in bytes: $VolumeSize "@ Write-ObservabilityLog -Message $volumeInfo } function Write-ObservabilityVolumeCreationStopTelemetry { param( [string] [Parameter(Mandatory=$true)] $ComputerName, [string] [Parameter(Mandatory=$true)] $Message, [string] [Parameter(Mandatory=$false)] $ExceptionDetails = "" ) [Microsoft.AzureStack.Observability.ObservabilityConfigTelemetryEventSource]::Log.ObservabilityVolumeCreationStop($ComputerName, $Message, $ExceptionDetails) if($ExceptionDetails) { Write-ObservabilityErrorLog -Message $Message } else { Write-ObservabilityLog -Message $Message } } function Write-ObservabilityVolumeMountedByScheduledTaskTelemetry { param( [string] [Parameter(Mandatory=$true)] $ComputerName, [string] [Parameter(Mandatory=$true)] $VolumeFilePath, [bool] [Parameter(Mandatory=$true)] $Success, [Parameter(Mandatory=$false)] $ExceptionDetails = "" ) [Microsoft.AzureStack.Observability.ObservabilityConfigTelemetryEventSource]::Log.ObservabilityVolumeMountedByScheduledTask( $ComputerName, $VolumeFilePath, $Success, $ExceptionDetails ) $message = @" Observability Volume Mounted by Scheduled Task ComputerName: $ComputerName Volume file path: $VolumeFilePath Success: $Success Exception Details (if applicable): $ExceptionDetails "@ if($ExceptionDetails) { Write-ObservabilityErrorLog -Message $Message } else { Write-ObservabilityLog -Message $Message } } function Write-ObservabilityVolumePrunerFileDeletionTelemetry { param( [string] [Parameter(Mandatory=$true)] $ComputerName, [string] [Parameter(Mandatory=$true)] $DirectoryPath, [int64] [Parameter(Mandatory=$true)] $DirectorySize, [string] [Parameter(Mandatory=$true)] $FilePath, [int64] [Parameter(Mandatory=$true)] $FileSize, [bool] [Parameter(Mandatory=$true)] $Success, [Parameter(Mandatory=$false)] $ExceptionDetails = "" ) [Microsoft.AzureStack.Observability.ObservabilityConfigTelemetryEventSource]::Log.ObservabilityVolumePrunerFileDeletion( $ComputerName, $DirectoryPath, $DirectorySize, $FilePath, $FileSize, $Success, $ExceptionDetails ) $message = @" Observability Volume Pruner File Deletion ComputerName: $ComputerName Directory path: $DirectoryPath Directory size in bytes: $DirectorySize File path: $FilePath File Size in bytes: $FileSize Success: $Success Exception Details (if applicable): $ExceptionDetails "@ if($success) { Write-ObservabilityLog $message } else { Write-ObservabilityErrorLog $message } } function Write-BootstrapNodeIdAndHardwareIdHashTelemetry { param( # Node ID assigned to node before cluster joining [string] [parameter(Mandatory=$true)] $BootstrapNodeId ) $hardwareHashId = Get-HardwareIdHash [Microsoft.AzureStack.Observability.ObservabilityConfigTelemetryEventSource]::Log.BootstrapNodeIdAndHardwareIdHash( $BootstrapNodeId, $hardwareHashId ) Write-ObservabilityLog "Bootstrap Node ID: $BootstrapNodeId Hardware Hash ID: $hardwareHashId" } function Add-VolumeFolderQuotaStatisticsScheduledTask { param( # Folder access path for observability drive [string] [parameter(Mandatory=$true)] $AccessPath, # Name of subfolder config file to read [string] [Parameter(Mandatory=$true)] $SubFolderConfigFileName ) $taskName = "ObservabilityVolumeQuotaStatistics" if(Test-ScheduledTaskExists -TaskName $taskName) { Unregister-ScheduledTask -TaskName $taskName -Confirm:$false | Out-Null } # Setup scheduled task to mount VHD on startup $observabilityNugetPath = "$PSScriptRoot\..\.." $scriptPath = "$observabilityNugetPath\content\Powershell\WriteObservabilityVolumeQuotaStatistics.ps1" Write-ObservabilityLog "Creating new scheduled task $taskName." $frequency = New-TimeSpan -Hours 6 $trigger = New-ScheduledTaskTrigger -RepetitionInterval $frequency -Once -At "12:00 AM" $action = New-ScheduledTaskAction ` -Execute "powershell.exe" ` -Argument "-Command $scriptPath -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName" $settings = New-ScheduledTaskSettingsSet $principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount $task = New-ScheduledTask -Action $action -Trigger $trigger -Settings $settings -Principal $principal Register-ScheduledTask -TaskName $taskName -TaskPath "Microsoft\AzureStack\Observability" -InputObject $task Write-ObservabilityLog "Creating new scheduled task $taskName succeeded." } function Install-ArcForServerAgent { param( [string] [parameter(Mandatory=$true)] $AgentMsiPath, [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $TenantId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $ResourceName, [string] [Parameter(Mandatory=$true)] $EnvironmentName, [string] [Parameter(Mandatory=$true)] $Region, [string] [Parameter(Mandatory=$false)] $ProxyUrl ) # NOTE: This Add-Type will not work if called from TelemetryAndDiagnostics ARC extension $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Add-Type -Path "$observabilityNugetPath\lib\net472\Microsoft.AzureStack.Observability.ObservabilityCommon.dll" $timestamp = [DateTime]::Now.ToString("yyyyMMdd-HHmmss") $logPath = (New-Item -Path "$env:LocalRootFolderPath" -ItemType Directory -Force).FullName $logFile = Join-Path -Path $logPath -ChildPath "ArcForServerInstall_${timestamp}.txt" $AgentWebLink = $ObservabilityConfigConstants.ArcForServerAgentWebLink $AgentMsiPath = Join-Path -Path $AgentMsiPath -ChildPath $ObservabilityConfigConstants.ArcForServerMsiFileName $programFiles = [Environment]::GetFolderPath([System.Environment+SpecialFolder]::ProgramFiles) $AgentExePath = Join-Path -Path $programFiles -ChildPath $ObservabilityConfigConstants.ArcForServerExePath Write-ObservabilityLog "$env:COMPUTERNAME Starting Arc-for-server agent install AgentWebLink: $AgentWebLink AgentMsiPath: $AgentMsiPath AgentExePath: $AgentExePath logs: $logFile" $res = $false try { $arcContext = New-Object Microsoft.AzureStack.Observability.ObservabilityCommon.ArcForServer.ArcContext $arcContext.AccessToken = $AccessToken $arcContext.SubscriptionId = $SubscriptionId $arcContext.ResourceGroup = $ResourceGroupName $arcContext.Location = $Region $arcContext.Cloud = $EnvironmentName $arcContext.ResourceName = $ResourceName $arcContext.TenantId = $TenantId Write-ObservabilityLog "AgentWebLink: $AgentWebLink AgentMsiPath: $AgentMsiPath AgentExePath: $AgentExePath" $arcAgent = New-Object Microsoft.AzureStack.Observability.ObservabilityCommon.ArcForServer.ArcAgent if ($ProxyUrl) { $res = $arcAgent.Onboard($arcContext, $AgentWebLink, $AgentMsiPath, $logFile, $AgentExePath, $ProxyUrl) } else { $res = $arcAgent.Onboard($arcContext, $AgentWebLink, $AgentMsiPath, $logFile, $AgentExePath) } } catch { Write-ObservabilityErrorLog "$env:COMPUTERNAME Failed to install/configure ArcAgent Exception: $_" } finally { if (Test-Path $AgentMsiPath) { Remove-Item -Path $AgentMsiPath -Verbose -ErrorAction Ignore } } Write-ObservabilityLog "Arc-for-server agent install $env:COMPUTERNAME. Status $res" if($res -eq $false ) { throw "Arc Agent failed to install or connect. Please check the Observability eventlog on the host" } } function Uninstall-ArcForServerAgent { param( [string] [parameter(Mandatory=$true)] $AgentMsiPath, [string] [Parameter(Mandatory=$true)] $AccessToken ) # NOTE: This Add-Type will not work if called from TelemetryAndDiagnostics ARC extension $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Add-Type -Path "$observabilityNugetPath\lib\net472\Microsoft.AzureStack.Observability.ObservabilityCommon.dll" $timestamp = [DateTime]::Now.ToString("yyyyMMdd-HHmmss") $logPath = (New-Item -Path "$env:LocalRootFolderPath" -ItemType Directory -Force).FullName $logFile = Join-Path -Path $logPath -ChildPath "ArcForServerUninstall_${timestamp}.txt" $AgentMsiPath = Join-Path -Path $AgentMsiPath -ChildPath $ObservabilityConfigConstants.ArcForServerMsiFileName $programFiles = [Environment]::GetFolderPath([System.Environment+SpecialFolder]::ProgramFiles) $AgentExePath = Join-Path -Path $programFiles -ChildPath $ObservabilityConfigConstants.ArcForServerExePath Write-ObservabilityLog "Starting Arc-for-server agent uninstall on $env:COMPUTERNAME. AgentMsiPath: $AgentMsiPath AgentExePath: $AgentExePath logs: $logFile" $arcContext = New-Object Microsoft.AzureStack.Observability.ObservabilityCommon.ArcForServer.ArcContext $arcContext.AccessToken = $AccessToken Write-ObservabilityLog "AgentMsiPath: $AgentMsiPath AgentExePath: $AgentExePath" $arcAgent = New-Object Microsoft.AzureStack.Observability.ObservabilityCommon.ArcForServer.ArcAgent $res = $arcAgent.Offboard($arcContext, $logFile, $AgentExePath, $AgentMsiPath) Write-ObservabilityLog "Arc-for-server agent uninstall $env:COMPUTERNAME. Status $res" return $res } function Update-ArcForServerAgent { param( [string] [parameter(Mandatory=$true)] $AgentMsiPath ) # NOTE: This Add-Type will not work if called from TelemetryAndDiagnostics ARC extension $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Add-Type -Path "$observabilityNugetPath\lib\net472\Microsoft.AzureStack.Observability.ObservabilityCommon.dll" $timestamp = [DateTime]::Now.ToString("yyyyMMdd-HHmmss") $logPath = (New-Item -Path "$env:LocalRootFolderPath" -ItemType Directory -Force).FullName $logFile = Join-Path -Path $logPath -ChildPath "ArcForServerUpdate_${timestamp}.txt" $AgentWebLink = $ObservabilityConfigConstants.ArcForServerAgentWebLink $AgentMsiPath = Join-Path -Path $AgentMsiPath -ChildPath $ObservabilityConfigConstants.ArcForServerMsiFileName Write-ObservabilityLog "$env:COMPUTERNAME Starting Arc-for-server agent update AgentWebLink: $AgentWebLink AgentMsiPath: $AgentMsiPath logs: $logFile" $res = $false try { Write-ObservabilityLog "AgentWebLink: $AgentWebLink AgentMsiPath: $AgentMsiPath" $arcAgent = New-Object Microsoft.AzureStack.Observability.ObservabilityCommon.ArcForServer.ArcAgent $res = $arcAgent.Update($AgentWebLink, $AgentMsiPath, $logFile) } catch { Write-ObservabilityErrorLog "$env:COMPUTERNAME Failed to update ArcAgent Exception: $_" } finally { if (Test-Path $AgentMsiPath) { Remove-Item -Path $AgentMsiPath -Verbose -ErrorAction Ignore } } Write-ObservabilityLog "Arc-for-server agent update $env:COMPUTERNAME. Status $res" if($res -eq $false ) { throw "Arc Agent failed to update. Please check the Observability eventlog on the host" } } function Connect-ArcForServerAgent { param( [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $TenantId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $ResourceName, [string] [Parameter(Mandatory=$true)] $EnvironmentName, [string] [Parameter(Mandatory=$true)] $Region, [string] [Parameter(Mandatory=$false)] $ProxyUrl ) # NOTE: This Add-Type will not work if called from TelemetryAndDiagnostics ARC extension $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Add-Type -Path "$observabilityNugetPath\lib\net472\Microsoft.AzureStack.Observability.ObservabilityCommon.dll" $programFiles = [Environment]::GetFolderPath([System.Environment+SpecialFolder]::ProgramFiles) $AgentExePath = Join-Path -Path $programFiles -ChildPath $ObservabilityConfigConstants.ArcForServerExePath Write-ObservabilityLog "$env:COMPUTERNAME Starting Arc-for-server agent connection AgentExePath: $AgentExePath" $res = $false try { $arcContext = New-Object Microsoft.AzureStack.Observability.ObservabilityCommon.ArcForServer.ArcContext $arcContext.AccessToken = $AccessToken $arcContext.SubscriptionId = $SubscriptionId $arcContext.ResourceGroup = $ResourceGroupName $arcContext.Location = $Region $arcContext.Cloud = $EnvironmentName $arcContext.ResourceName = $ResourceName $arcContext.TenantId = $TenantId Write-ObservabilityLog "AgentExePath: $AgentExePath" $arcAgent = New-Object Microsoft.AzureStack.Observability.ObservabilityCommon.ArcForServer.ArcAgent if($ProxyUrl) { $res = $arcAgent.Connect($arcContext, $AgentExePath, $ProxyUrl) } else { $res = $arcAgent.Connect($arcContext, $AgentExePath) } } catch { Write-ObservabilityErrorLog "$env:COMPUTERNAME Failed to connect ArcAgent Exception: $_" } Write-ObservabilityLog "Arc-for-server agent connection for $env:COMPUTERNAME. Status $res" if($res -eq $false ) { throw "Arc Agent failed to connect. Please check the Observability eventlog on the host" } } function Get-RequiredVersionFromValidatedRecipe { param ( [string] [Parameter(Mandatory=$true)] $ModuleName ) $vcNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.VersionControl.Operations" Import-Module $vcNugetPath\content\Scripts\VersionControlHelpers.psm1 $requiredVersion = Get-ValidatedRecipeModuleVersion($ModuleName) Trace-Execution "$ModuleName version obtained from validated receipe is $requiredVersion" return $requiredVersion } function Install-ExtensionPreRequisites { Write-ObservabilityLog "$env:COMPUTERNAME Installing pre-requisites for Arc-for-server extension." try { Set-PSRepository -Name 'PSGallery' -InstallationPolicy Trusted if(-not (Get-Module -Name "Az.Accounts" -ListAvailable)) { $requiredVersion = Get-RequiredVersionFromValidatedRecipe -ModuleName "Az.Accounts" Write-Verbose "Installing Az.Accounts module with version: $requiredVersion" Install-Module "Az.Accounts" -RequiredVersion $requiredVersion -Verbose } if(-not (Get-Module -Name "Az.ConnectedMachine" -ListAvailable)) { $requiredVersion = Get-RequiredVersionFromValidatedRecipe -ModuleName "Az.ConnectedMachine" Write-Verbose "Installing Az.ConnectedMachine module with version: $requiredVersion" Install-Module "Az.ConnectedMachine" -RequiredVersion $requiredVersion -Verbose } Set-PSRepository -Name 'PSGallery' -InstallationPolicy Untrusted } catch { Write-ObservabilityErrorLog "$env:COMPUTERNAME Failed to install pre-requisites for Arc-for-server Extension Exception: $_" throw $_ } Write-ObservabilityLog "$env:COMPUTERNAME Finished pre-requisites Arc-for-server extension." } function Install-ArcForServerExtensions { param( [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $TenantId, [string] [Parameter(Mandatory=$true)] $AccountId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $ResourceName, [string] [Parameter(Mandatory=$true)] $EnvironmentName, [string] [Parameter(Mandatory=$true)] $Region, [string] [Parameter(Mandatory=$false)] $Type = $ObservabilityConfigConstants.ExtensionType ) Write-ObservabilityLog "$env:COMPUTERNAME Starting Arc-for-server extension install." Write-ObservabilityLog "$env:COMPUTERNAME Install required modules." Install-ExtensionPreRequisites Import-Module Az.Accounts -Force Import-Module Az.ConnectedMachine -Force Write-ObservabilityLog "$env:COMPUTERNAME Authenticating to Azure with Access token. $($AccessToken.Length)" Connect-AzAccount -AccessToken $AccessToken -SubscriptionId $SubscriptionId -AccountId $AccountId $publisher = $ObservabilityConfigConstants.ExtensionPublisher Write-ObservabilityLog "$env:COMPUTERNAME Checking if [$Type] extension is already installed." $extension = Get-AzConnectedMachineExtension -ResourceGroupName $ResourceGroupName ` -MachineName $ResourceName ` -ErrorAction SilentlyContinue ` | Where-Object {$Type -eq $_.MachineExtensionType} # Install the extension only if it not present or publisher doesnt match if($null -eq $extension -or $extension.Publisher -ne $publisher) { Write-ObservabilityLog "$env:COMPUTERNAME Going to install [$Type] extension." New-AzConnectedMachineExtension -Name $Type ` -ResourceGroupName $ResourceGroupName ` -MachineName $ResourceName ` -Location $Region ` -Publisher $publisher ` -ExtensionType $Type ` -ErrorAction Stop Write-ObservabilityLog "$env:COMPUTERNAME Checking the installed [$Type] extension version." $extension = Get-AzConnectedMachineExtension -Name $Type ` -ResourceGroupName $ResourceGroupName ` -MachineName $ResourceName ` -ErrorAction SilentlyContinue if($null -eq $extension -or $extension.ProvisioningState -ne "Succeeded" -or $extension.Publisher -ne $publisher) { throw "$env:COMPUTERNAME extension installation failed. $($extension.Name) ProvisioningState: $($extension.ProvisioningState) Publisher: $($extension.Publisher)" } } # Fail the interface if extension is not found in good state if($null -eq $extension -or $extension.ProvisioningState -ne "Succeeded" -or $extension.Publisher -ne $publisher) { throw "$env:COMPUTERNAME Extension installation found failed. $($extension.Name) ProvisioningState: $($extension.ProvisioningState) Publisher: $($extension.Publisher)" } Write-ObservabilityLog "$env:COMPUTERNAME Finished Arc-for-server extension install. Status $($extension.ProvisioningState)" } function Uninstall-ArcForServerExtensions { param( [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $AccountId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $CloudId ) Write-ObservabilityLog "$env:COMPUTERNAME Arc-for-server agent extension uninstall start" Import-Module Az.Accounts -Force Import-Module Az.ConnectedMachine -Force $ResourceName = $env:COMPUTERNAME + "-" + $CloudId Write-ObservabilityLog "$env:COMPUTERNAME Authenticating to Azure with Access token." Connect-AzAccount -AccessToken $AccessToken -SubscriptionId $SubscriptionId -AccountId $AccountId $extensionName = $ObservabilityConfigConstants.ExtensionType Write-ObservabilityLog "$env:COMPUTERNAME Checking if $extensionName extension is already installed on resource $ResourceName." $extension = Get-AzConnectedMachineExtension -Name $extensionName ` -ResourceGroupName $ResourceGroupName ` -MachineName $ResourceName ` -ErrorAction SilentlyContinue if($null -ne $extension) { Write-ObservabilityLog "$env:COMPUTERNAME Starting $extensionName extension uninstallation on resource $ResourceName" Remove-AzConnectedMachineExtension -MachineName $ResourceName ` -Name $extensionName ` -ResourceGroupName $ResourceGroupName ` -SubscriptionId $SubscriptionId Write-ObservabilityLog "$env:COMPUTERNAME Checking if $extensionName extension if it is uninstalled on resource $ResourceName." $extension = Get-AzConnectedMachineExtension -Name $extensionName ` -ResourceGroupName $ResourceGroupName ` -MachineName $ResourceName ` -ErrorAction SilentlyContinue if($null -ne $extension) { throw "$env:COMPUTERNAME $extensionName extension uninstallation failed. Name: $($extension.Name) ProvisioningState: $($extension.ProvisioningState) Publisher: $($extension.Publisher)" } } Write-ObservabilityLog "$env:COMPUTERNAME Arc-for-server agent extension uninstall complete" } function Invoke-CachedTelemetryFilesParsing { param( [string] [Parameter(Mandatory=$true)] $LogOrchestratorNugetPath, [string] [Parameter(Mandatory=$true)] $CorrelationId ) $result = $false try { $transcriptFileName = "{0}.{1:yyyy-MM-dd}.log" -f "Invoke-CachedTelemetryFilesParsing", $(Get-Date) $transcriptFilePath = Join-Path -Path "$env:LocalRootFolderPath" -ChildPath $transcriptFileName Start-Transcript -Path $transcriptFilePath -Append | Out-Null Write-ObservabilityLog "$env:COMPUTERNAME Cached Telemetry Files Parsing start" $parsingEnginePath = Join-Path -Path $LogOrchestratorNugetPath -ChildPath $ObservabilityConfigConstants.ParsingEngineInternalPath $configFilePath = Join-Path -Path $observabilityNugetPath -ChildPath $ObservabilityConfigConstants.TelemetryCacheConfigurationInternalPath Write-ObservabilityLog "$env:COMPUTERNAME CachedTelemetryFilesParsing parsingEnginePath: $parsingEnginePath configFilePath: $configFilePath" $telemetryCompnentsCache = Get-Content $configFilePath | ConvertFrom-Json Write-ObservabilityLog "$env:COMPUTERNAME Found Components count $($telemetryCompnentsCache.Components) from $configFilePath." foreach($component in $telemetryCompnentsCache.Components) { $cacheLogPath = $component.CachePath $exceptionMsg = "" $pathFound = $false try { Write-ObservabilityLog "$env:COMPUTERNAME Processing Telemetry cache for Component Name: $($component.Name) RegistryPath: $($component.RegistryPath) Path $cacheLogPath." if(-not [string]::IsNullOrEmpty($component.RegistryPath)) { Write-ObservabilityLog "$env:COMPUTERNAME Going to resolve registry path for Component Name: $($component.Name) RegistryPath: $($component.RegistryPath)." $registryProperty = Get-ItemProperty -Path $ObservabilityConfigConstants.TelemetryRegistryPath -Name $component.RegistryPath -ErrorAction SilentlyContinue if($registryProperty) { $registryLogPath = $registryProperty.$($component.RegistryPath) } Write-ObservabilityLog "$env:COMPUTERNAME Going to resolve registry path for Component Name: $($component.Name) RegistryPath: $($component.RegistryPath) Resolved Path $registryLogPath." $cacheLogPath = $registryLogPath } Write-CachedTelemetryParsingStartTelemetry -ComponentName $component.Name -TelemetryCachePath $cacheLogPath -RegistryPath $component.RegistryPath Write-ObservabilityLog "$env:COMPUTERNAME Processing Telemetry cache for Component Name: $($component.Name) Path: $cacheLogPath." $cachePathJson = ConvertTo-Json -InputObject $cacheLogPath $parsingEngineArg = 'IngestionInfo:{"GenerateTelemetryEvents":"true","LogDirectoryPath":' + $cachePathJson + ',"TracingContext":"' + $CorrelationId + '"}' $pathFound = if (($cacheLogPath) -and (Test-Path -Path $cacheLogPath) ) { $true } else { $false } if($pathFound) { $currentDir = [System.IO.Path]::GetDirectoryName($parsingEnginePath) Write-ObservabilityLog "$env:COMPUTERNAME Going to start ParsingEngine $parsingEnginePath with Arguments: $parsingEngineArg CurrentDirectory: $currentDir." Start-Process -FilePath $parsingEnginePath -NoNewWindow -Wait -ArgumentList $parsingEngineArg -Passthru -WorkingDirectory $currentDir Write-ObservabilityLog "$env:COMPUTERNAME ParsingEngine finished execution for component $($component.Name) Going to read result file from $currentDir" $resultFilePath = Join-Path -Path $currentDir -ChildPath $ObservabilityConfigConstants.ParsingEngineResultFileName Write-ObservabilityLog "$env:COMPUTERNAME ParsingEngine execution result file for component $component is $resultFilePath" $parsingEngineResultStr = Get-Content $resultFilePath Write-ObservabilityLog "$env:COMPUTERNAME ParsingEngine execution result for component $($component.Name) is $parsingEngineResultStr" $parsingEngineResult = $parsingEngineResultStr | ConvertFrom-Json $result = $parsingEngineResult.Failed -eq $false } else { $result = $true Write-ObservabilityLog "$env:COMPUTERNAME Component $($component.Name) Telemetry cache path $cacheLogPath not found. So skipping parsing" } } catch { $exceptionMsg = $_.Exception.Message.ToString() } finally { Write-CachedTelemetryParsingStopTelemetry -ComponentName $component.Name ` -TelemetryCachePath $cacheLogPath ` -RegistryPath $component.RegistryPath ` -PathFound $pathFound ` -ParserResult $parsingEngineResultStr ` -ExceptionDetails $exceptionMsg } } } finally { Stop-Transcript | Out-Null Write-ObservabilityLog "$env:COMPUTERNAME Cached Telemetry Files Parsing end Result: $result" } return $result } function Write-ArcForServerInstallationStartTelemetry { param( [string] [Parameter(Mandatory=$true)] $ComputerNames ) [Microsoft.AzureStack.Observability.ObservabilityConfigTelemetryEventSource]::Log.ArcForServerInstallationStart( $ComputerNames ) $info = @" Starting Arc-for-Server agent installation ComputerNames: $ComputerNames "@ Write-ObservabilityLog -Message $info } function Write-ArcForServerInstallationStopTelemetry { param( [string] [Parameter(Mandatory=$true)] $ComputerName, [string] [Parameter(Mandatory=$true)] $Message, [string] [Parameter(Mandatory=$false)] $ExceptionDetails = "" ) [Microsoft.AzureStack.Observability.ObservabilityConfigTelemetryEventSource]::Log.ArcForServerInstallationStop($ComputerName, $Message, $ExceptionDetails) if($ExceptionDetails) { Write-ObservabilityErrorLog -Message $Message } else { Write-ObservabilityLog -Message $Message } } function Get-Sha256Hash { param( [string] [Parameter(Mandatory=$true)] $ClearString ) $hasher = [System.Security.Cryptography.HashAlgorithm]::Create('sha256') $hash = $hasher.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($ClearString)) $hashString = [System.BitConverter]::ToString($hash) $hashString = $hashString.Replace('-', '') return $hashString } function Get-HardwareIdHash { $computerInfo = Get-ComputerInfo $manufacturer = $computerInfo.CsManufacturer $model = $computerInfo.CsModel $serialNumber = $computerInfo.BiosSeralNumber return (Get-Sha256Hash -ClearString "$manufacturer-$model-$serialNumber").toLower() } function Set-GMAScenarioRegistryKeyToBootstrap { # Import GMATenantJsonHelper $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.GenevaMonitoringAgent") -ChildPath "content" Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking # $MiscConstants imported from GMATenantJsonHelper $gmaScenarioRegKeyPath = $MiscConstants.GMAScenarioRegKey.Path $gmaScenarioRegKeyName = $MiscConstants.GMAScenarioRegKey.Name $gmaScenarioBootstrapValue = $MiscConstants.GMAScenarioRegKey.Bootstrap $gmaScenarioRegKeyPropertyType = $MiscConstants.GMAScenarioRegKey.PropertyType if(-not (Test-Path $gmaScenarioRegKeyPath)) { Write-ObservabilityLog "Creating GMAScenario registry path at $gmaScenarioRegKeyPath" New-Item -Path $gmaScenarioRegKeyPath -Force } New-ItemProperty ` -Path $gmaScenarioRegKeyPath ` -Name $gmaScenarioRegKeyName ` -PropertyType $gmaScenarioRegKeyPropertyType ` -Value $gmaScenarioBootstrapValue ` -Force Write-ObservabilityLog "Set $gmaScenarioRegKeyName at $gmaScenarioRegKeyPath to $gmaScenarioBootstrapValue" } function Write-CachedTelemetryParsingStartTelemetry { param( [string] [Parameter(Mandatory=$true)] $ComponentName, [string] [Parameter(Mandatory=$false)] $TelemetryCachePath, [string] [Parameter(Mandatory=$false)] $RegistryPath ) [Microsoft.AzureStack.Observability.ObservabilityConfigTelemetryEventSource]::Log.CachedTelemetryParsingStart( $ComponentName, $TelemetryCachePath ) $info = @" Starting Cached Telemetry file parsing ComponentName: $ComponentName TelemetryCachePath: $TelemetryCachePath RegistryPath: $RegistryPath "@ Write-ObservabilityLog -Message $info } function Write-CachedTelemetryParsingStopTelemetry { param( [string] [Parameter(Mandatory=$true)] $ComponentName, [string] [Parameter(Mandatory=$false)] $TelemetryCachePath, [string] [Parameter(Mandatory=$false)] $RegistryPath, [bool] [Parameter(Mandatory=$true)] $PathFound, [string] [Parameter(Mandatory=$false)] $ParserResult, [string] [Parameter(Mandatory=$false)] $ExceptionDetails = "" ) [Microsoft.AzureStack.Observability.ObservabilityConfigTelemetryEventSource]::Log.CachedTelemetryParsingStop($ComponentName, $TelemetryCachePath, $PathFound, $ParserResult, $ExceptionDetails) $info = @" Finished Cached Telemetry file parsing ComponentName: $ComponentName TelemetryCachePath: $TelemetryCachePath RegistryPath: $RegistryPath PathFound: $PathFound ParserResult: $ParserResult ExceptionDetails: $ExceptionDetails "@ if($ExceptionDetails) { Write-ObservabilityErrorLog -Message $info } else { Write-ObservabilityLog -Message $info } } function Get-ArcResourceId { param( [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $CloudId ) $resourceName = Get-ArcResourceName Write-ObservabilityLog "Generating Arc resource id with SubscriptionId $SubscriptionId ResourceGroupName $ResourceGroupName ResourceName $resourceName" $arcResourceIdFormat = $ObservabilityConfigConstants.ArcForServerResourceIdFormat $arcResourceId = [string]::Format($arcResourceIdFormat, $SubscriptionId, $ResourceGroupName, $resourceName) Write-ObservabilityLog "Get-ArcResourceId returning $arcResourceId" return $arcResourceId } function Get-ArcResourceName { # Resouce name has to be host name for device registration to work $resourceName = hostname Write-ObservabilityLog "Get-ArcResourceName returning $resourceName" return $resourceName } # Start Arc Remote Support agent if the EdgeRemoteSupport extension has been in listener mode. # Remove listener mode reg key # No-op if extension was not in listener mode function StartArcRemoteSupportAgent { $success = $false $paths = (Get-ChildItem "C:\Packages\Plugins\Microsoft.AzureStack.Observability*EdgeRemoteSupport*\*\" -ErrorAction SilentlyContinue).FullName $extError = "" foreach ($extRootPath in $paths) { Write-ObservabilityLog "Found EdgeRemoteSupport extension at $extRootPath." if ($success) { Write-ObservabilityLog "Arc Extension Remote Support Agent already started. Ignoring extension at $extRootPath." } else { try { Import-Module (Join-Path -Path $extRootPath -ChildPath 'Common\ExtensionCommon.psd1') ` -DisableNameChecking ` -Force ` -Verbose:$false $constantsName = "RemoteSupportConstants" $RemoteSupportConstants = Get-Constants -Constants $constantsName $logFile = Get-HandlerLogFile -Constants $constantsName # Check if extension has been in listener mode $listenerModeRegKeyPath = $RemoteSupportConstants.ListenerModeRegKey.Path $listenerModeRegKeyName = $RemoteSupportConstants.ListenerModeRegKey.Name $listenerModeRegKeyValue = (Get-ItemProperty -Path $listenerModeRegKeyPath -ErrorAction SilentlyContinue).$listenerModeRegKeyName if ($listenerModeRegKeyValue -eq 1) { Write-ObservabilityLog "Bringing EdgeRemoteSupport extension out of listener mode." # Register Remote Support Agent Write-ObservabilityLog "Registering Remote Support Agent" Register-RemoteSupportAgent -LogFile $logFile # Start Remote Support Agent Write-ObservabilityLog "Starting Remote Support Agent" Start-RemoteSupportAgent -LogFile $logFile # Removing listener mode registry key Remove-RegistryKey ` -path $RemoteSupportConstants.ListenerModeRegKey.Path ` -Name $RemoteSupportConstants.ListenerModeRegKey.Name ` -LogFile $LogFile Write-ObservabilityLog "Remote Support Agent successfully registered and started." $success = $true } else { Write-ObservabilityLog "EdgeRemoteSupport Extension was not in listener mode." $success = $true } } catch { Write-ObservabilityErrorLog "Starting Arc Remote Support Agent failed with error $_" $extError = $_ } } } if (-not $success) { if ($extError) { throw "Starting Arc Remote Support Agent failed with error $extError" } else { $errMsg = "Could not start Arc Remote Support Agent because no EdgeRemoteSupport Extension was found at C:\Packages\Plugins." Write-ObservabilityErrorLog $errMsg throw $errMsg } } } function Test-IsCIEnv { $CIRegKey = @{ Path = 'HKLM:\Software\Microsoft\SQMClient\' Name = 'IsCIEnv' } $ObsCIRegKey = @{ Path = 'HKLM:\Software\Microsoft\AzureStack\Observability\' Name = 'IsCIEnv' PropertyType = 'DWORD' } $ciKeyExists = $null -ne (Get-ItemProperty -Path $CIRegKey.Path -Name $ObsCIRegKey.Name -ErrorAction SilentlyContinue) $obsCiKeyExists = $null -ne (Get-ItemProperty -Path $ObsCIRegKey.Path -Name $ObsCIRegKey.Name -ErrorAction SilentlyContinue) if ($ciKeyExists -and -not $obsCiKeyExists) { Write-ObservabilityLog "Registry key $($CIRegKey.Name) exists at path $($CIRegKey.Path). Copying to path $($ObsCIRegKey.Path)." $ciKeyValue = (Get-ItemProperty -Path $CIRegKey.Path -Name $CIRegKey.Name).$($CIRegKey.Name) if (-not (Test-Path -Path $ObsCIRegKey.Path)) { New-Item ` -Path $ObsCIRegKey.Path ` -Force ` -Verbose:$false ` | Out-Null } New-ItemProperty ` -Path $ObsCIRegKey.Path ` -Name $ObsCIRegKey.Name ` -PropertyType $ObsCIRegKey.PropertyType ` -Value $ciKeyValue ` -Force ` | Out-Null Write-ObservabilityLog "Created Registry key $($ObsCIRegKey.Name) at path $($ObsCIRegKey.Path) with value $ciKeyValue." } return $ciKeyExists -or $obsCiKeyExists } Export-ModuleMember -Function Add-MountVolumeScheduledTask Export-ModuleMember -Function Add-ObservabilityVolumeWithRetry Export-ModuleMember -Function Add-VolumeAccessPath Export-ModuleMember -Function Add-VolumeFolderQuotaStatisticsScheduledTask Export-ModuleMember -Function Get-ObservabilityFolders Export-ModuleMember -Function Get-Sha256Hash Export-ModuleMember -Function Remove-ObservabilityFolderOldestFiles Export-ModuleMember -Function Set-FolderQuotas Export-ModuleMember -Function Set-GMAScenarioRegistryKeyToBootstrap Export-ModuleMember -Function New-VolumeFoldersAndPrunerWithRetry Export-ModuleMember -Function Test-IsCIEnv Export-ModuleMember -Function Write-ObservabilityErrorLog Export-ModuleMember -Function Write-ObservabilityLog Export-ModuleMember -Function Write-BootstrapNodeIdAndHardwareIdHashTelemetry Export-ModuleMember -Function Write-ObservabilityVolumeCreationStartTelemetry Export-ModuleMember -Function Write-ObservabilityVolumeCreationStopTelemetry Export-ModuleMember -Function Write-ObservabilityVolumeMountedByScheduledTaskTelemetry Export-ModuleMember -Function Write-ObservabilityVolumePrunerFileDeletionTelemetry Export-ModuleMember -Function Install-ArcForServerAgent Export-ModuleMember -Function Uninstall-ArcForServerAgent Export-ModuleMember -Function Update-ArcForServerAgent Export-ModuleMember -Function Connect-ArcForServerAgent Export-ModuleMember -Function Uninstall-ArcForServerExtensions Export-ModuleMember -Function Install-ArcForServerExtensions Export-ModuleMember -Function Write-ArcForServerInstallationStartTelemetry Export-ModuleMember -Function Write-ArcForServerInstallationStopTelemetry Export-ModuleMember -Function Get-ArcResourceId Export-ModuleMember -Function Get-ArcResourceName Export-ModuleMember -Function Invoke-CachedTelemetryFilesParsing Export-ModuleMember -Function StartArcRemoteSupportAgent # SIG # Begin signature block # MIIoOwYJKoZIhvcNAQcCoIIoLDCCKCgCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAm6F0lt3obk7Qd # AAls1cvmbEJyXEWpx08Ci+h2uVqou6CCDYUwggYDMIID66ADAgECAhMzAAADri01 # UchTj1UdAAAAAAOuMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMxMTE2MTkwODU5WhcNMjQxMTE0MTkwODU5WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQD0IPymNjfDEKg+YyE6SjDvJwKW1+pieqTjAY0CnOHZ1Nj5irGjNZPMlQ4HfxXG # yAVCZcEWE4x2sZgam872R1s0+TAelOtbqFmoW4suJHAYoTHhkznNVKpscm5fZ899 # QnReZv5WtWwbD8HAFXbPPStW2JKCqPcZ54Y6wbuWV9bKtKPImqbkMcTejTgEAj82 # 6GQc6/Th66Koka8cUIvz59e/IP04DGrh9wkq2jIFvQ8EDegw1B4KyJTIs76+hmpV # M5SwBZjRs3liOQrierkNVo11WuujB3kBf2CbPoP9MlOyyezqkMIbTRj4OHeKlamd # WaSFhwHLJRIQpfc8sLwOSIBBAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUhx/vdKmXhwc4WiWXbsf0I53h8T8w # VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzUwMTgzNjAfBgNVHSMEGDAW # gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw # MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx # XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB # AGrJYDUS7s8o0yNprGXRXuAnRcHKxSjFmW4wclcUTYsQZkhnbMwthWM6cAYb/h2W # 5GNKtlmj/y/CThe3y/o0EH2h+jwfU/9eJ0fK1ZO/2WD0xi777qU+a7l8KjMPdwjY # 0tk9bYEGEZfYPRHy1AGPQVuZlG4i5ymJDsMrcIcqV8pxzsw/yk/O4y/nlOjHz4oV # APU0br5t9tgD8E08GSDi3I6H57Ftod9w26h0MlQiOr10Xqhr5iPLS7SlQwj8HW37 # ybqsmjQpKhmWul6xiXSNGGm36GarHy4Q1egYlxhlUnk3ZKSr3QtWIo1GGL03hT57 # xzjL25fKiZQX/q+II8nuG5M0Qmjvl6Egltr4hZ3e3FQRzRHfLoNPq3ELpxbWdH8t # Nuj0j/x9Crnfwbki8n57mJKI5JVWRWTSLmbTcDDLkTZlJLg9V1BIJwXGY3i2kR9i # 5HsADL8YlW0gMWVSlKB1eiSlK6LmFi0rVH16dde+j5T/EaQtFz6qngN7d1lvO7uk # 6rtX+MLKG4LDRsQgBTi6sIYiKntMjoYFHMPvI/OMUip5ljtLitVbkFGfagSqmbxK # 7rJMhC8wiTzHanBg1Rrbff1niBbnFbbV4UDmYumjs1FIpFCazk6AADXxoKCo5TsO # zSHqr9gHgGYQC2hMyX9MGLIpowYCURx3L7kUiGbOiMwaMIIHejCCBWKgAwIBAgIK # YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm # aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw # OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD # VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG # 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la # UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc # 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D # dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ # lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk # kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 # A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd # X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL # 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd # sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 # T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS # 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI # bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL # BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD # uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF # BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h # cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA # YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn # 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 # v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b # pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ # KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy # CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp # mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi # hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb # BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS # oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL # gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX # cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGgwwghoIAgEBMIGVMH4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p # Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAAOuLTVRyFOPVR0AAAAA # A64wDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIOre # qTbpkOlaWl99pGwCsbOm0tqJFJAGpyxW5jtEucTwMEIGCisGAQQBgjcCAQwxNDAy # oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20wDQYJKoZIhvcNAQEBBQAEggEAqrYrIij9FoRWFQGVn0M1fDBIC01Cq5k68PBs # KBHRHuGI8phmEJkaPZKtoXfFia6J7XNdDwafFW8mBK0/ONjUuIBRgCzmp+KezP5g # laFLXS7dPOiYS2Bh9osnuBWbWNch6u6QsYn2FpEpKeUZYO8/+noIVEMoUeKkvdqF # g4LQjbNVfpzfNSNnxa+nqecrUjRCAFozLW8283XnmzuZ0TlUUGvr/P3X13y0ZUHo # FIFqm8vzfl886AWed4pxfJPXwrfJ8AN0GzeehcEq/oe1UtYVhXfBEdZwTVPpoFdr # 6dzY2YETjtvY7vVKjC+VJML+4XYKsmB9pYTgs0PB1i0YbfM2EqGCF5YwgheSBgor # BgEEAYI3AwMBMYIXgjCCF34GCSqGSIb3DQEHAqCCF28wghdrAgEDMQ8wDQYJYIZI # AWUDBAIBBQAwggFRBgsqhkiG9w0BCRABBKCCAUAEggE8MIIBOAIBAQYKKwYBBAGE # WQoDATAxMA0GCWCGSAFlAwQCAQUABCDuHvfpQ107e2Yto8BDHIRK0cRWfajLEnfE # W4put8cMoQIGZeeoQK+HGBIyMDI0MDMxMTE4MTgzOC4zMlowBIACAfSggdGkgc4w # gcsxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJTAjBgNVBAsT # HE1pY3Jvc29mdCBBbWVyaWNhIE9wZXJhdGlvbnMxJzAlBgNVBAsTHm5TaGllbGQg # VFNTIEVTTjpFMDAyLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUt # U3RhbXAgU2VydmljZaCCEe0wggcgMIIFCKADAgECAhMzAAAB7gXTAjCymp2nAAEA # AAHuMA0GCSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNo # aW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29y # cG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEw # MB4XDTIzMTIwNjE4NDU0NFoXDTI1MDMwNTE4NDU0NFowgcsxCzAJBgNVBAYTAlVT # MRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQK # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJTAjBgNVBAsTHE1pY3Jvc29mdCBBbWVy # aWNhIE9wZXJhdGlvbnMxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVTTjpFMDAyLTA1 # RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC # AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAL7xvKXXooSJrzEpLi9UvtEQ # 45HsvNgItcS1aB6rI5WWvO4TP4CgJri0EYRKNsdNcQJ4w7A/1M94popqV9NTldIa # OkmGkbHn1/EwmhNhY/PMPQ7ZECXIGY4EGaIsNdENAkvVG24CO8KIu6VVB6I8jxXv # 4eFNHf3VNsLVt5LHBd90ompjWieMNrCoMkCa3CwD+CapeAfAX19lZzApK5eJkFNt # Tl9ybduGGVE3Dl3Tgt3XllbNWX9UOn+JF6sajYiz/RbCf9rd4Y50eu9/Aht+TqVW # rBs1ATXU552fa69GMpYTB6tcvvQ64Nny8vPGvLTIR29DyTL5V+ryZ8RdL3Ttjus3 # 8dhfpwKwLayjJcbc7AK0sDujT/6Qolm46sPkdStLPeR+qAOWZbLrvPxlk+OSIMLV # 1hbWM3vu3mJKXlanUcoGnslTxGJEj69jaLVxvlfZESTDdas1b+Nuh9cSz23huB37 # JTyyAqf0y1WdDrmzpAbvYz/JpRkbYcwjfW2b2aigfb288E72MMw4i7QvDNROQhZ+ # WB3+8RZ9M1w9YRCPt+xa5KhW4ne4GrA2ZFKmZAPNJ8xojO7KzSm9XWMVaq2rDAJx # pj9Zexv9rGTEH/MJN0dIFQnxObeLg8z2ySK6ddj5xKofnyNaSkdtssDc5+yzt74l # syMqZN1yOZKRvmg3ypTXAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQUEIjNPxrZ3CCe # vfvF37a/X9x2pggwHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD # VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j # cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG # CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw # MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD # CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAHdnIC9rYQo5ZJWk # GdiTNfx/wZmNo6znvsX2jXgCeH2UrLq1LfjBeg9cTJCnW/WIjusnNlUbuulTOdrL # af1yx+fenrLuRiQeq1K6AIaZOKIGTCEV9IHIo8jTwySWC8m8pNlvrvfIZ+kXA+ND # Bl4joQ+P84C2liRPshReoySLUJEwkqB5jjBREJxwi6N1ZGShW/gner/zsoTSo9CY # BH1+ow3GMjdkKVXEDjCIze01WVFsX1KCk6eNWjc/8jmnwl3jWE1JULH/yPeoztot # Iq0PM4RQ2z5m2OHOeZmBR3v8BYcOHAEd0vntMj2HueJmR85k5edxiwrEbiCvJOyF # TobqwBilup0wT/7+DW56vtUYgdS0urdbQCebyUB9L0+q2GyRm3ngkXbwId2wWr/t # dUG0WXEv8qBxDKUk2eJr5qeLFQbrTJQO3cUwZIkjfjEb00ezPcGmpJa54a0mFDlk # 3QryO7S81WAX4O/TmyKs+DR+1Ip/0VUQKn3ejyiAXjyOHwJP8HfaXPUPpOu6TgTN # zDsTU6G04x/sMeA8xZ/pY51id/4dpInHtlNcImxbmg6QzSwuK3EGlKkZyPZiOc3O # cKmwQ9lq3SH7p3u6VFpZHlEcBTIUVD2NFrspZo0Z0QtOz6cdKViNh5CkrlBJeOKB # 0qUtA8GVf73M6gYAmGhl+umOridAMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ # mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT # Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m # dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh # dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1 # WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB # BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK # NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg # fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp # rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d # vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9 # 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR # Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu # qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO # ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb # oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6 # bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t # AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW # BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb # UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku # aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA # QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2 # VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu # bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw # LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93 # d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt # MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q # XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6 # U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt # I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis # 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp # kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0 # sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e # W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ # sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7 # Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0 # dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ # tB1VM1izoXBm8qGCA1AwggI4AgEBMIH5oYHRpIHOMIHLMQswCQYDVQQGEwJVUzET # MBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMV # TWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmlj # YSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046RTAwMi0wNUUw # LUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2WiIwoB # ATAHBgUrDgMCGgMVAIijptU29+UXFtRYINDdhgrLo76ToIGDMIGApH4wfDELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9z # b2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwDQYJKoZIhvcNAQELBQACBQDpmWZ+MCIY # DzIwMjQwMzExMTExNjE0WhgPMjAyNDAzMTIxMTE2MTRaMHcwPQYKKwYBBAGEWQoE # ATEvMC0wCgIFAOmZZn4CAQAwCgIBAAICIFoCAf8wBwIBAAICE6MwCgIFAOmat/4C # AQAwNgYKKwYBBAGEWQoEAjEoMCYwDAYKKwYBBAGEWQoDAqAKMAgCAQACAwehIKEK # MAgCAQACAwGGoDANBgkqhkiG9w0BAQsFAAOCAQEAbZl6gKd4lNVEmvwSi2PiIg1M # Uhj7DccrIIP4P4LnJ9MOaJqU1p3XuunyZTtVdBh+clR3ZetSDF8O1wTgfMaNcP9v # m/PiRUY7X2dVY8+pTipKKHnB4Xfk68UiLicREAnrYT0yQrfRmRezT6yqNqHO10zK # 3fa6MScUqeayYK6ikQxIzR/7fyiVcVIvZ5sg0Mybga0JGr8XikzIaCwCzii+bESQ # VtTe2jroE6tbRX98e4Qzs8TC0SFpM6dlWhhOdLC3Ptio4ydHr7Gvt6v3fgnrYM2J # FSX2AYbMIYEBLZuV9YRwXrkCjz5ifhc75RVTVTH4el600BsQTprnbaJldPxv2TGC # BA0wggQJAgEBMIGTMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u # MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp # b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB # 7gXTAjCymp2nAAEAAAHuMA0GCWCGSAFlAwQCAQUAoIIBSjAaBgkqhkiG9w0BCQMx # DQYLKoZIhvcNAQkQAQQwLwYJKoZIhvcNAQkEMSIEILKxt/X59wZf+2VdqScQ1BlH # Hho1lVSuCL/BvYWab3KwMIH6BgsqhkiG9w0BCRACLzGB6jCB5zCB5DCBvQQgT1B3 # FJWF+r5V1/4M+z7kQiQHP2gJL85B+UeRVGF+MCEwgZgwgYCkfjB8MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQg # VGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAe4F0wIwspqdpwABAAAB7jAiBCBxzSKK # li55wdM5EMY/M4Als8mJz3vS/bIZsbHfF0/vJjANBgkqhkiG9w0BAQsFAASCAgB8 # +1YS0BkZ+7dFBwJyeW9qJ4F+CJk8Z61g/Mmamw31LyMbXVpKNJgCQu7sEFNTRoDM # HC3sZ3341GOLUC2P39USpQzQBhaJohSMFOlw6iFyX7IBL+6r96daXayM2koKnUIs # uACjgnRv7PZo58ku7PS5PQCsDrp1ZgyFq0jIPZXS/ZK8GrpJRT6scjmAAUvYSiUW # NnSCMBDXaWgmjiuYCwOCEkfyrEdeo+Cx+L/Dms6dMj9uo2L5nDBffSFNwPiMQRGx # iB23vrsqjyG30A+dnMOJosr4xeCytapPuSyOFyWuyGq3ukAVqUtzN0Ri1Gma0rKk # YXNShlqxLvDYPU4Dj9JFQGPZ/pMA+cMXqRWfQshsDvLC09kORYezKIzxY5nmTCP3 # IjD/TLVjCiL4okJer5nogTOF07czxyZAMk0qCjzzBq6hmThbZpft5qiHV/gE3wbc # kQ6D1ys5U04a7/d3PRQY4yaR5cP0GJscKuwnEeWsAwqNITTfxmtC4DNdM3mEcUj2 # tJl65jSUPaiHQNxWi5AzkDuwPiDOiZZvVa6JB0ApHxn8q1JkZ8aKuBPDLQ9WLijI # P7fKqEmT9h0Fkpb8MtOsPcln1Pei/Owa66hVxXYRtNQs23YoirNY7bbfBpFQMBpA # iSgXJpuDSu6Z13KTZ3INIWm+CBtVoUpHg6dnb3xNdg== # SIG # End signature block |