Azs.TeamsIntegration.psm1
|
#Requires -Modules Azs.Update.Admin, Azs.Management Import-Module -Name Azs.Management -Force Import-Module -Name Azs.Update.Admin -Force # Import modules $Location = $PSScriptRoot [System.Reflection.Assembly]::LoadFrom("$Location\Dependencies\Antlr4.Runtime.dll") [System.Reflection.Assembly]::LoadFrom("$Location\Dependencies\Newtonsoft.Json.dll") [System.Reflection.Assembly]::LoadFrom("$Location\Dependencies\Jurassic.dll") [System.Reflection.Assembly]::LoadFrom("$Location\Dependencies\AdaptiveCards.dll") [System.Reflection.Assembly]::LoadFrom("$Location\Dependencies\AdaptiveExpressions.dll") [System.Reflection.Assembly]::LoadFrom("$Location\Dependencies\AdaptiveCards.Templating.dll") $TeamsMessageContent = @" { "type": "message", "attachments": [ { "contentType": "application/vnd.microsoft.card.adaptive", "contentUrl": null, "content": {{MessageContent}} } ] } "@ function Send-AzsUpdate { <# .SYNOPSIS Send Update status to operators and audiences. .DESCRIPTION Retrieves update status and post a webrequest to operator and audience URIs, optionally as an adaptive card for teams. .EXAMPLE PS C:\> $OperatorUri = "https://outlook.office.com/webhook/<etc>" PS C:\> $AudienceUri = "https://outlook.office.com/webhook/<etc>" PS C:\> $Bridge = "https://teams.microsoft.com/l/meetup-join/<etc>" PS C:\> $stamp = "Prod" PS C:\> Send-AzsUpdate -AudienceUri $AudienceUri -OperatorUri $OperatorUri -Stamp $stamp -BridgeInformation $Bridge Updates audience every 10 minutes and operators (default) every 5 minutes for stamp "Prod" .NOTES Operators are assumed to have access to the audience channel and thus do not need rich data on the update. Operators recieve update status and duration and messages about the execution of retrieving and building status updates for all. OperatorFrequency must be less than or equal AudienceFrequency, it is assumes operators need update status quicker than audiences. .PARAMETER OperatorFrequency How often (approximately) the monitor the update for Operators. Must be 5, 10, 15, 20, 30 or 60 .PARAMETER AudienceFrequency How often (approximately) the monitor the update for Audiences. Must be 5, 10, 15, 20, 30 or 60 .PARAMETER AudienceUri Uri(s) to send Audience status updates .PARAMETER OperatorUri Uri(s) to send Operator status updates .PARAMETER Stamp An Azs.Management construct that is passed to PEP and ARM connections. Stamps must be onboarded in Azs.Management using Add-Stamp. .PARAMETER PepCredential An Azs.Management construct that is passed to PEP connections. Cloud Admin credential for PEP connections. .PARAMETER UpdateStatus NOT YET IMPLEMENT. FOR OFFLINE UPDATES .PARAMETER StampInformation NOT YET IMPLEMENT. FOR OFFLINE UPDATES .PARAMETER DisposePEP Close the PEP Session after every iteration. .PARAMETER Log Saves zip of diagnostic data for troubleshooting .PARAMETER LogPath Saves zip of diagnostic data for troubleshooting .PARAMETER RetrySeconds Wait in seconds for retries connecting to ARM and PEP. Default is 30 .NOTES v0.1.1 - Send-AzsUpdate errors when Get-AzsUpdate returns nothing v0.1.1 Not using -log can lead to errors #> [CmdletBinding()] [Alias("Send-AzsUpdateStatus")] param ( [Parameter(Mandatory = $false, HelpMessage = 'Will override connectionUri in StampDefinition')] [System.Uri[]] $AudienceUri, [Parameter(Mandatory = $false, HelpMessage = 'Send issues to a seperate Operator URI')] [System.Uri[]] $OperatorUri, [Parameter(Mandatory = $true, ParameterSetName = 'PEP')] [ArgumentCompleter( { (Get-Stamp).Name | Sort-Object })] [string] $Stamp, [Parameter(Mandatory = $false, ParameterSetName = 'PEP')] [pscredential] $PepCredential, [Parameter(Mandatory = $true, ParameterSetName = 'File', HelpMessage = 'Use offline XML status of update')] [xml] $UpdateStatus, [Parameter(Mandatory = $true, ParameterSetName = 'File', HelpMessage = 'Use offline Stamp Information JSON status of update')] [psobject] $StampInformation, [Parameter(Mandatory = $false, ParameterSetName = 'PEP', HelpMessage = 'Force PEP disconnect after every run.')] [switch]$disposePEP, [Parameter(Mandatory = $false, HelpMessage = 'Saves zip of diagnostic data for troubleshooting')] [switch]$log, [Parameter(Mandatory = $false, HelpMessage = 'Customize log path')] [string]$logPath = (Join-Path "$HOME" ".AzsMgmtTeams"), [Parameter(Mandatory = $false, HelpMessage = 'Wait in seconds for retries connecting to ARM and PEP. Default is 30')] [int]$retrySeconds = 30, [Parameter(Mandatory = $false, HelpMessage = 'Optionally include bridge information on the card')] $BridgeInformation ) try { $logPath = Initialize-LogPath -LogPath $logPath if ($PSCmdlet.ParameterSetName -ne 'File') { $pepsession = Get-PepSessionWithRetries -stamp $stamp -retrySeconds $retrySeconds -PepCredential $PepCredential if (-not $pepSession) { throw "Cannot establish PEP session. Ensure Get-PepSession -stamp $stamp works." } Write-CustomLog "[$Stamp] Getting Stamp Information" $StampInformation = Invoke-PepCommand -PepSession $pepSession -scriptBlock { Get-AzureStackStampInformation } if ($StampInformation) { $ShowStampInfo = $true } # get update status from URP and determine latest update state $updatesFromURP = Get-AzsUpdateWithRetries -stamp $stamp -retrySeconds $retrySeconds Write-CustomLog "[$Stamp] Get lastest Update from URP" $latestUpdateFromURP = $updatesFromURP | Select-Object -first 1 Write-CustomLog "[$Stamp] Update $($latestUpdateFromURP.DisplayName) is $($latestUpdateFromURP.state)" if (-not $latestUpdateFromURP) { Write-CustomLog "[$Stamp] No Updates in progress according URP. This can be transient." $showHistory = $false } else { $Banner = "$($latestUpdateFromURP.DisplayName) - $($latestUpdateFromURP.State.ToString())" $latestUpdateRun = Get-AzsUpdateRun -UpdateName $latestUpdateFromURP.Name | ` Sort-Object @{Expression = { [DateTime]$_.TimeStarted } } -Descending | ` Select-Object -first 1 } if ($BridgeInformation) { $showBridge = $true } else { $showBridge = $false } # Update running show Update-Progress card if ($latestUpdateFromURP.state -in 'Installing', 'Preparing') { Write-CustomLog "Select template Azs-TeamsIntegration-Update-InProgress.json" $UpdateCardTemplate = Get-Content "$PSScriptRoot\Templates\Azs-TeamsIntegration-Update-InProgress.json" -Raw $updateStartTime = $latestUpdateRun.TimeStarted $totalDuration = Format-TimeSpan ([System.Xml.XmlConvert]::ToTimeSpan($latestUpdateRun.duration)) # Only connect to PEP if update is "installing" and there is an Audience. if ($latestUpdateFromURP.state -in 'Installing' -and $AudienceUri) { Write-CustomLog "[$Stamp] Getting Update Steps from PEP" try { [xml]$updateStatus = Invoke-PepCommand -PepSession $pepSession -ScriptBlock { Get-AzureStackUpdateStatus } $UpdateSummary = Get-UpdateSummary -UpdateStatus $updateStatus $updateProgress = $UpdateSummary.updateProgress $summary = $UpdateSummary.summary } catch { if ($_.Exception.Message -match 'The session state is Broken|No valid sessions were specified') { $OperatorMessage = "[$Stamp] Update is {0} but PEP Session lost. This can happen during update. Adaptive card may be missing progress, and will be restored when possible. {1}" -f $latestUpdateFromURP.state, $_.Exception.Message } else { $OperatorMessage = $_.Exception.Message } Send-TextMessage -uri $OperatorUri -message $OperatorMessage } } $showProgress = if ($updateProgress.StepsInProgress) { $true } else { $false } # Do not show progress (steps) but show duration and status #$showProgress = if ($latestUpdateFromURP.state -eq 'Installing') { $true } else { $false } $showPrepProgress = if ($latestUpdateFromURP.state -eq 'Preparing') { $true } else { $false } $PrepProgress = $latestUpdateRun | Select-Object -expand ProgressStep | Select-Object Name, Status Write-CustomLog "[$Stamp] Steps in progress:" $PrepProgress | ForEach-Object { Write-CustomLog (("[$Stamp] {0}" -f $_.Name).PadRight(75) + $_.Status) } Write-CustomLog "[$Stamp] Update Duration: $totalDuration" } elseif ($latestUpdateFromURP.State -in 'Ready', 'Installed', 'PreparationFailed', 'InstallationFailed') { # Show update not running card but if there are previous updates, show the previous updates. # this is essential a "success" card, or "you ran this too soon" card # to add banner and duration to history Write-CustomLog "Select template Azs-TeamsIntegration-Update-NotRunning.json" $UpdateCardTemplate = Get-Content "$PSScriptRoot\Templates\Azs-TeamsIntegration-Update-NotRunning.json" -Raw $showProgress = $false if (-not $updatesFromURP) { $showHistory = $false } else { $showHistory = $true $updateHistory = $updatesFromURP | Select-Object Location, DisplayName, @{label = 'State'; Expression = { $_.State.tostring() } }, VersionNumber } if ($latestUpdateFromURP.State -eq 'Installed') { $showDuration = $true $totalDurationFinal = Format-TimeSpan ([System.Xml.XmlConvert]::ToTimeSpan($latestUpdateRun.duration)) } else { $showDuration = $false } # the previous run errored, we want logs. # TO DO, ensure this is not noisy. if ($latestUpdateFromURP.State -in 'PreparationFailed', 'InstallationFailed') { $collectAzsLogs = $true } } else { Write-CustomLog "[$Stamp] URP return unknown state: $($latestUpdateFromURP.state) : $($latestUpdateFromURP.DisplayName)" } } else { Write-CustomLog "[$Stamp] Skipping PEP connection" } # Gather cmdlet used and version $moduleVersion = $MyInvocation.MyCommand.Module.Version $moduleName = $MyInvocation.MyCommand.Module.Name $commandletName = (Get-PSCallStack)[-2].Command # Create PSObject of progress data in "i hate nulls and ints" mode $UpdateData = New-Object -TypeName PSObject -Property @{ Banner = if ($Banner) { $Banner } else { "Azure Stack Update" } ShowScopedRepair = if ($UpdateProgress.ShowScopedRepair) {$true} else {$false} showPrepProgress = $showPrepProgress ShowStampInfo = $ShowStampInfo showDuration = $showDuration totalDurationFinal = $totalDurationFinal Stamp = New-Object -TypeName PSObject -Property @{ AdminPortal = $StampInformation.AdminExternalEndpoints.AdminPortal TenantPortal = $StampInformation.TenantExternalEndpoints.TenantPortal Prefix = $StampInformation.Prefix Hardware = $StampInformation.HardwareOEM NumberOfNodes = "$($StampInformation.NumberOfNodes)" Version = $StampInformation.StampVersion CloudId = $StampInformation.CloudId } Update = New-Object -TypeName PSObject -Property @{ Prep = if ($showPrepProgress) {$PrepProgress} else {''} Summary = if ($Summary) { $Summary } else { '' } UpdateProgress = if ($UpdateProgress -and $showProgress) { $UpdateProgress } else { '' } UpdateHistory = if ($updateHistory) { $UpdateHistory } else { '' } UpdateName = if ($latestUpdateRun.Name) { $latestUpdateRun.Name } else { '' } } ShowHistory = $showHistory ShowProgress = $showProgress ShowBridge = $showBridge BridgeInformation = $BridgeInformation commandletName = if ($commandletName) { $commandletName } else { '' } moduleVersion = if ($moduleVersion) { "$moduleVersion" } else { '' } moduleName = if ($moduleName) { $moduleName } else { '' } } # Get Update Detail Adaptive Card Template if there's an audience. if ($AudienceUri) { $TeamsMessageContent = Format-TeamsMessage -JsonTemplate $UpdateCardTemplate -AdaptiveCardData $UpdateData Send-MessageContent -uri $AudienceUri -MessageContent $TeamsMessageContent } else { Write-CustomLog "[$Stamp] Skipping Adaptive card. No Audience." } } catch { $OperatorMessage = "We hit a problem sending data from the stamp to teams: Error: {0}" -f $_.exception.message Send-TextMessage -uri $OperatorUri -message $OperatorMessage } finally { if ($pepSession -and $disposePEP) { Write-CustomLog "[$Stamp] Closing Pepsession" Close-PepSession -Stamp $Stamp } else { Write-CustomLog "[$Stamp] Leaving Pepsession open." } if ($collectAzsLogs) { $pepsession = Get-PepSession -stamp $stamp -PepCredential $PepCredential Write-CustomLog "[$Stamp] Sending Azure Stack Logs." $pepsession = Get-PepSessionWithRetries -stamp $stamp -retrySeconds $retrySeconds -PepCredential $PepCredential Invoke-PepCommand -PepSession $pepSession -ScriptBlock { Send-AzureStackDiagnosticLog -FitlerByRole SeedRingServices } -AsJob Send-TextMessage -uri $OperatorUri -message "[$Stamp] Sending Azure Stack Logs." } if ($updatesFromURP) { $updatesFromURP | ConvertTo-Json -depth 15 | Out-File $logPath\AllUpdates.json } if ($StampInformation) { $StampInformation | ConvertTo-Json -depth 15 | Out-File $logPath\StampInformation.json } if ($UpdateStatus) { $UpdateStatus | Out-File $logPath\UpdateStatus.xml } if ($UpdateData) { ($UpdateData | ConvertTo-Json -Depth 20) | Out-File $logPath\UpdateData.json } if ($latestUpdateRun) { $latestUpdateRun | ConvertTo-Json -depth 15 | Out-File $logPath\UpdateRun.json } if ($TeamsMessageContent) { $TeamsMessageContent | Out-File $logPath\TeamsMessageContent.json } if ($log) { # send operator the log if logging on, highlight important messaging $TeamsMessageContent = Format-OperatorMessage -message (Get-Content (Join-Path $logPath AzsMgmtTeams.log)) } foreach ($connector in $OperatorUri) { Invoke-RestMethod -Uri $connector -Method Post -Body $TeamsMessageContent -ContentType 'application/json' } $zipFile = "{0}\AzMgmtTeamsLog_{1}.zip" -f ((Split-Path $logPath -Parent), (Split-Path $logPath -Leaf)) Write-CustomLog "[$Stamp] Zipping logs. $zipFile" Compress-Archive -Path $logPath -DestinationPath $zipFile Write-CustomLog "[$Stamp] Removing $logPath" Remove-Item -Path $logPath -Recurse -Force -ErrorAction SilentlyContinue } } function Format-TimeSpan { <# .SYNOPSIS Convert timespan to Hours and Minutes #> param ([TimeSpan]$timeSpan) $h = $TimeSpan.Days * 24 + $TimeSpan.Hours $m = $TimeSpan.Minutes if ($h -gt 0) { "$h hours $m minutes" } else { "{0:d2} minutes" -f $m } } # Builds an 'AdaptiveCard' from the given template and data Function Transform-AdaptiveCard { <# .SYNOPSIS Builds an 'AdaptiveCard' from the given template and data #> [OutputType([AdaptiveCards.AdaptiveCard])] Param ( [Parameter( Mandatory = $True, ValueFromPipeline = $True, ValueFromPipelineByPropertyName = $True)] [ValidateNotNullOrEmpty()] [String] $JsonTemplate, [Parameter( Mandatory = $True, ValueFromPipeline = $True, ValueFromPipelineByPropertyName = $True)] [ValidateNotNullOrEmpty()] [String] $JsonData, [Parameter()] [Switch] $AsString ) $AdaptiveTransformer = [AdaptiveCards.Templating.AdaptiveCardTemplate]::new($JsonTemplate) $AdaptiveEvaluationContext = [AdaptiveCards.Templating.EvaluationContext]::new($JsonData) $CardJson = $AdaptiveTransformer.Expand($AdaptiveEvaluationContext) $cardjson | Out-File -FilePath $LogPath\MidTransform.json If ($PSBoundParameters.ContainsKey('AsString')) { Write-Output -InputObject $CardJson } Else { $AdaptiveCard = ([AdaptiveCards.AdaptiveCard]::FromJson($CardJson)).Card Write-Output -InputObject $AdaptiveCard } } function Write-CustomLog { <# .SYNOPSIS Write log messages to file and verbose stream. #> [CmdletBinding()] param ($message) $message = "[{0}] {1}" -f (Get-Date).ToUniversalTime().ToString('s'), $message Write-Verbose -Message $message $message | Out-File -FilePath (Join-Path $logPath AzsMgmtTeams.log) -Append } function Get-PepSessionWithRetries { <# .SYNOPSIS Attempt to get PEPSession with retries #> param ($stamp, $retrySeconds = 30, $PepCredential) Clear-Variable retry, pepSession -ErrorAction SilentlyContinue while ($retry -lt 3 -and !$pepSession) { $retry++ Write-CustomLog "[$Stamp] Opening Pepsession. Attempt $retry/3" try { $pepSession = Connect-PepSession -Stamp $Stamp -PEPCredential $PepCredential } catch { Write-CustomLog ("[$Stamp] Pep Attempt failed. Reason {0}" -f $_.exception.message) } if ($pepSession) { return $pepSession } else { Start-Sleep -seconds $retrySeconds } } } function Get-AzsUpdateWithRetries { <# .SYNOPSIS Attempt to get update status from ARM with retries #> param ($stamp, $retrySeconds) try { Write-CustomLog "[$Stamp] Connecting to Admin ARM" Connect-AzureStack -Stamp $stamp } catch { Write-Warning -Message ("Connecting to Azure Stack failed. Ensure Connect-AzureStack $Stamp works. Error: {0}" -f $_.Exception.message) } Clear-Variable retry, updatesFromURP -ErrorAction SilentlyContinue while ($retry -lt 3 -and !$updatesFromURP) { $retry++ Write-CustomLog "[$Stamp] Getting Update from URP. Attempt $retry/3" try { # If there's an update we only need to return 1 update, other return an ordered list. $updatesFromURP = Get-AzsUpdate | Where-Object State -match 'Installing|Preparing' if (-not $updatesFromURP) { $updatesFromURP = Get-AzsUpdate | Sort-Object @{Expression = { [System.Version]$_.Version } } -Descending } if ($updatesFromURP) { # write a new property called VersionNumber to include OEM version combined view $updatesFromURP = $updatesFromURP | Select-object *,@{label='VersionNumber';expression={ if ($null -eq $_.version -and $null -ne $_.OemVersion) {$_.OemVersion}else{$_.version} } } $updatesFromURP | Foreach-Object { Write-CustomLog ("[$Stamp] {0} {1} {2} {3}" -f ` $_.DisplayName, $_.State, $_.VersionNumber, $_.InstalledDate) } return $updatesFromURP } else { Start-Sleep -seconds $retrySeconds } } catch { Write-CustomLog ("[$Stamp] Failed getting Update from URP. This can be transient during updates. Error: {0}" -f $_.exception.message) } } } function Initialize-LogPath { <# .SYNOPSIS Determine timestamped log path and return value #> param ($logPath,[switch]$root) if (-not $root) { $logPath = Join-Path $logPath (Get-Date).ToString('yyyyMMddHHmmss') } New-Item $logPath -ItemType Directory -Force -ErrorAction SilentlyContinue | Out-Null return $logPath } function Send-MessageContent { <# .SYNOPSIS Send message content #> param ([system.uri[]]$uri, $messageContent) if ($uri) { Write-CustomLog ("[$Stamp] Posting message content to Uri(s): {0}" -f ($uri -join ',')) } else { Write-CustomLog "[$Stamp] No message to post right now" } foreach ($u in $uri) { Invoke-RestMethod -Uri $u -Method Post -Body $messageContent -ContentType 'application/json' } } function Format-TeamsMessage { <# .SYNOPSIS Transform data and adaptive card template and embed in teams message. #> param ($JsonTemplate, $AdaptiveCardData) Write-CustomLog "[$Stamp] Tranforming Adaptive Card" $UpdateProgressAdaptiveCard = Transform-AdaptiveCard -JsonTemplate $UpdateCardTemplate -JsonData ($AdaptiveCardData | ConvertTo-Json -Depth 20) -AsString $TeamsMessageContent = $TeamsMessageContent.replace('{{MessageContent}}', $UpdateProgressAdaptiveCard) return $TeamsMessageContent } function Get-UpdateSummary { <# .SYNOPSIS Build update PSObject. #> param ($UpdateStatus) # All Steps $steps = $updateStatus.SelectNodes("//Step") | Select-Object FullStepIndex, Name, Status, StartTimeUtc, EndTimeUtc #Steps Success $Success = $updateStatus.SelectNodes("//Step") | Where-Object { $_.Status -eq 'Success' } | Select-Object FullStepIndex, Name, Status, StartTimeUtc, EndTimeUtc #Steps in Progress $sbTinyTimeformat = { (Format-TimeSpan (([datetime]::Now) - [datetime]$_.StartTimeUtc)).replace(' hours ', 'h:').replace(' minutes', 'm') } $stepsSb = { if ($_.ExecutionContext.Roles.Role.Nodes.Node.Name){ "{0} ({1})" -f $_.Name, $_.ExecutionContext.Roles.Role.Nodes.Node.Name } else { $_.Name} } $InProgress = $updateStatus.SelectNodes("//Step") | Where-Object Status -eq InProgress | Select-Object FullStepIndex, @{label='Name';Expression = $stepsSb }, Status, StartTimeUtc, EndTimeUtc, @{'label' = 'duration'; expression = $sbTinyTimeformat } #Steps in Error $InError = $updateStatus.SelectNodes("//Step") | Where-Object Status -eq Error | Select-Object FullStepIndex, Name, Status, StartTimeUtc, EndTimeUtc # Format Summary data $summary = New-Object -TypeName PSObject -Property @{ Completed = ($updateStatus.SelectNodes("//Step") | Where-Object Status -eq Success).count InProgress = ($updateStatus.SelectNodes("//Step") | Where-Object Status -eq InProgress).count Failed = ($updateStatus.SelectNodes("//Step") | Where-Object Status -eq Error).count } # Scoped Repair $scopedRepairTask = $UpdateStatus.SelectNodes("//RemediationAction") | Select-Object @{N="ActionType";E={"{0}" -f $_.Task.ActionType}}, @{N="ActionStatus";E={"{0}" -f $_.ActionStatus}} if ($scopedRepairTask.ActionStatus -in 'InProgress','Error','Cancelled') { $ShowScopedRepair = $true Write-CustomLog ("[$Stamp] Showing scoped repair status {0}" -f $scopedRepairTask.ActionStatus) } else { $ShowScopedRepair = $false Write-CustomLog ("[$Stamp] Not showing scoped repair status {0}" -f $scopedRepairTask.ActionStatus) } # Format InProgress data # if inprogress steps might exceed message limit, trim it. if ($InProgress.count -gt 15) { $InProgress = $InProgress | Where-Object { $_.FullStepIndex.split('.').count -gt 2 } | Sort-Object [datetime]$_.StartTimeUtc | Select-Object -first 15 # teams message limit, removed top level jobs and trim to 15 oldest jobs. } $UpdateProgress = New-Object -TypeName PSObject -Property @{ duration = $totalDuration status = $latestUpdateFromURP.State.ToString() StepsInProgress = $InProgress steps = $steps ShowScopedRepair = $ShowScopedRepair ScopedRepairStatus = $scopedRepairTask.ActionStatus } return (New-Object PSObject -Property @{summary = $summary; UpdateProgress = $UpdateProgress }) } Export-ModuleMember -Function Send-AzsUpdate function Watch-AzsUpdate { <# .SYNOPSIS Continually monitor Updates for changes and invoke updates to operators and audiences. .DESCRIPTION Runs in loop every minute for (configurable) totalMinutes, default 7200. Each iteration checks if the operators or the audiences should updated as per OperatorFreqency and AudienceFrequency respectively. If Operators should be updated, the status of the most recent update is retrieved and changes (name, duration, status) are determined. If Audiences should be updated and changes are detected, both audiences and operators receive an update. If Audiences are not to be updated, i.e. update has changed but AudienceFrequency (a update) isn't due, the operators get a short status update. .EXAMPLE PS C:\> $OperatorUri = "https://outlook.office.com/webhook/<etc>" PS C:\> $AudienceUri = "https://outlook.office.com/webhook/<etc>" PS C:\> $Bridge = "https://teams.microsoft.com/l/meetup-join/<etc>" PS C:\> $stamp = "Prod" PS C:\> Watch-AzsUpdate -AudienceUri $AudienceUri -OperatorUri $OperatorUri -Stamp $stamp -BridgeInformation $Bridge -AudienceFrequency 30 Updates audience every 30 minutes and operators (default) every 5 minutes for stamp "Prod" .NOTES Operators are assumed to have access to the audience channel and thus do not need rich data on the update. Operators recieve update status and duration and messages about the execution of retrieving and building status updates for all. OperatorFrequency must be less than or equal AudienceFrequency, it is assumes operators need update status quicker than audiences. .PARAMETER OperatorFrequency How often (approximately) the monitor the update for Operators. Must be 5, 10 or 15. .PARAMETER AudienceFrequency How often (approximately) the monitor the update for Audiences. Must be 30 or 60. .PARAMETER AudienceUri Uri(s) to send Audience status updates .PARAMETER OperatorUri Uri(s) to send Operator status updates .PARAMETER Stamp An Azs.Management construct that is passed to PEP and ARM connections. Stamps must be onboarded in Azs.Management using Add-Stamp. .PARAMETER PepCredential An Azs.Management construct that is passed to PEP connections. Cloud Admin credential for PEP connections. .PARAMETER UpdateStatus NOT YET IMPLEMENT. FOR OFFLINE UPDATES .PARAMETER StampInformation NOT YET IMPLEMENT. FOR OFFLINE UPDATES .PARAMETER DisposePEP Close the PEP Session after every iteration. .PARAMETER Log Saves zip of diagnostic data for troubleshooting .PARAMETER LogPath Saves zip of diagnostic data for troubleshooting .PARAMETER RetrySeconds Wait in seconds for retries connecting to ARM and PEP. Default is 30 .PARAMETER TotalMinutes Total runtime in minutes. Defaults to 7200 minutes (5 days) .NOTES v0.1.1 Watch-AzsUpdate will fail when no updates are found using Get-AzsUpdate. Restart cmdlet when Get-AzsUpdate has output e.g. update ready, running, completed or failed. v0.1.1 Not using -log can lead to errors #> param ( [Parameter(Mandatory = $true, ParameterSetName = 'PEP')] [ArgumentCompleter( { (Get-Stamp).Name | Sort-Object })] [string] $Stamp, [Parameter(Mandatory = $false, ParameterSetName = 'PEP')] [pscredential] $PepCredential, [Parameter(Mandatory = $false, HelpMessage = 'Uri(s) to send Audience status updates')] [System.Uri[]] $AudienceUri, [Parameter(Mandatory = $false, HelpMessage = 'Uri(s) to send Audience status updates')] [System.Uri[]] $OperatorUri, [Parameter(Mandatory = $false, HelpMessage = 'How often (approximately) the monitor the update for Operators. Must be 5, 10 or 15')] [ValidateSet(5, 10, 15)] $OperatorFrequency = 5, [Parameter(Mandatory = $false, HelpMessage = 'How often (approximately) to send the update to the audiences. Must be 30 or 60')] [ValidateSet(30, 60)] $AudienceFrequency = 30, [Parameter(Mandatory = $false, HelpMessage = 'Optionally include bridge information on the card')] $BridgeInformation, [Parameter(Mandatory = $true, ParameterSetName = 'File', HelpMessage = 'Use offline XML status of update')] [xml] $UpdateStatus, [Parameter(Mandatory = $true, ParameterSetName = 'File', HelpMessage = 'Use offline Stamp Information JSON status of update')] [psobject] $StampInformation, [Parameter(Mandatory = $false, ParameterSetName = 'PEP', HelpMessage = 'Force PEP disconnect after every run.')] [switch]$DisposePEP, [Parameter(Mandatory = $false, HelpMessage = 'Saves zip of diagnostic data for troubleshooting')] [switch]$Log, [Parameter(Mandatory = $false, HelpMessage = 'Customize log path')] [string]$LogPath = (Join-Path "$HOME" ".AzsMgmtTeams"), [Parameter(Mandatory = $false, HelpMessage = 'Wait in seconds for retries connecting to ARM and PEP. Default is 30')] [int]$RetrySeconds = 30, [Parameter(Mandatory = $false, HelpMessage = 'Total runtime in minutes. Defaults to 7200 minutes (5 days)')] $totalMinutes = 7200 ) #New-EventLog -Source AzsTeamsIntegration -LogName Application -ErrorAction SilentlyContinue if ($AudienceFrequency -lt $OperatorFrequency) { throw "You must set OperatorFrequency <= AudienceFrequency." } $LogPath = Initialize-LogPath -logPath $logPath -Root $i = 0 $previousAudienceUpdate = 'NA' while ($i -le $totalMinutes) { try { # determine if operators and audiences should get an update $showOperator = if (($i % $OperatorFrequency) -eq 0) { $OperatorUri } else { $null } $showAudience = if (($i % $AudienceFrequency) -eq 0) { $AudienceUri } else { $null } if ($showOperator) { # Get Updates and latest update status $updates = Get-AzsUpdateWithRetries -stamp $stamp -retrySeconds $retrySeconds Write-CustomLog "[$Stamp] Get latest Update from URP" $latestUpdate = $updates | Select-Object -first 1 if ($latestUpdate) { # Get latest run and stringcat the in progress steps to use during status change detection, useful during preparation. $latestUpdateRun = Get-AzsUpdateRun -UpdateName $latestUpdate.Name | ` Sort-Object @{Expression = { [DateTime]$_.TimeStarted } } -Descending | ` Select-Object -first 1 | ` Select-Object *, @{label = 'StepInProgress'; Expression = { ($_.ProgressStep.Name) -join ',' } } } else { # if there's no updates i.e. new stamp latest version, we will update the operator only. # audience will only be updated if there's an update inprogress, ready etc.. throw "No updates found. Restart cmdlet when updates are ready, running or completed." } # determine if the audience has the latest update if (-not $latestUpdateRun -and $previousAudienceUpdate) { # no latest run (e.g. update 'Ready') $sendAudienceUpdate = $true # audience gets one update until the update is in progress } elseif ($latestUpdateRun -and $previousAudienceUpdate) { # there's a latest run (could be historical) and a previous update (could be initial 'NA') Write-CustomLog ("[$Stamp] UpdateName {0} State {1} Duration {2} Steps {3}" -f $latestUpdateRun.Name, $latestUpdateRun.State, $latestUpdateRun.Duration, $latestUpdateRun.StepInProgress) if (Compare-Object -ReferenceObject $latestUpdateRun -DifferenceObject $previousAudienceUpdate -Property Duration, Name, State, StepInProgress) { $sendAudienceUpdate = $true # runs differ, audience needs update, this could be last update "update completed" Write-CustomLog ("[$Stamp] LatestUpdateRun and PreviousAudienceUpdate differ => SendAudienceUpdate {0}" -f [bool]$SendAudienceUpdate) } else { $sendAudienceUpdate = $false # runs don't differ, audience has the latest update. Write-CustomLog ("[$Stamp] LatestUpdateRun and PreviousAudienceUpdate do not differ => SendAudienceUpdate {0}" -f [bool]$SendAudienceUpdate) } } elseif (-not $latestUpdateRun -and -not $previousAudienceUpdate) { # neither exist means Update 'Ready' but we've been through a loop (above) $sendAudienceUpdate = $false # audience doesn't need an update. } elseif ($latestUpdateRun -and -not $previousAudienceUpdate) { # edge case, two updates back to back Write-CustomLog ("[$Stamp] UpdateName {0} State {1} Duration {2} Steps {3}" -f $latestUpdate.Name, $latestUpdateRun.State, $latestUpdateRun.Duration, $latestUpdateRun.StepInProgress) $sendAudienceUpdate = $true } else { $sendAudienceUpdate = $false # something unexpected has happened } Write-CustomLog ("[$Stamp] LatestUpdateRun: {0}, PreviousUpdateRun {1} => SendAudienceUpdate {2}" -f [bool]$latestUpdateRun, [bool]$previousAudienceUpdate, [bool]$sendAudienceUpdate) # if an Audience Update is due provide one # and remember the state of the update if ($showAudience -and $sendAudienceUpdate) { $PSBoundParameters['AudienceUri'] = $showAudience $PSBoundParameters['OperatorUri'] = $showOperator $PSBoundParameters.Remove('OperatorFrequency') | Out-Null $PSBoundParameters.Remove('AudienceFrequency') | Out-Null Send-AzsUpdate @PSBoundParameters # remember what the audience just saw $previousAudienceUpdate = $latestUpdateRun | Select-Object Duration, Name, State, StepInProgress } else { # give short update to operator $PrepProgress = $latestUpdateRun | Select-Object -expand ProgressStep | Select-Object Name, Status $operatorMessage = @( "[$Stamp] Update $($latestUpdate.DisplayName) is $($latestUpdate.state)" ) $operatorMessage += $PrepProgress | ForEach-Object { ("[$Stamp] {0}" -f $_.Name).PadRight(75) + $_.Status } $OperatorMessageContent = Format-OperatorMessage -message $operatorMessage foreach ($opUri in $OperatorUri) { Invoke-RestMethod -Uri $opUri -Method Post -Body $OperatorMessageContent -ContentType 'application/json' } } } Write-CustomLog ("[$Stamp] Next cycle runs at {0}" -f [datetime]::now.addseconds(60)) #Write-EventLog LogName Application -Source "AzsTeamsIntegration" EntryType Information EventID 20500 Message "Monitoring for Azure Stack Updates" # loop every 1 minute and evaluate if operators and audience need updates. Start-Sleep -Seconds 60 $i++ } catch { if ($PepCredential) { $pepSession = Get-PepSessionWithRetries -stamp $Stamp -PepCredential $PepCredential } else { Get-PepSessionWithRetries -stamp $Stamp } $AzureStackUpdateStatus = Invoke-PepCommand -PEPSession $pepSession -Scriptblock { Get-AzureStackUpdateStatus -statusOnly } if ($AzureStackUpdateStatus.value -eq 'Failed') { $operatorMessage = "[$Stamp] UpdateStatus is {0}. Exiting Watch-AzsUpdate. Reason: {1}" -f $AzureStackUpdateStatus.value,$_.exception.message Send-TextMessage -Uri $OperatorUri -Message $operatorMessage throw $operatorMessage break } else { $operatorMessage = "[$Stamp] UpdateStatus is {0}. Warning during Watch-AzsUpdate. Reason: {0}. Re-evaluating in 1 minute." -f $_.exception.message Send-TextMessage -Uri $OperatorUri -Message $operatorMessage } } } } Export-ModuleMember -Function Watch-AzsUpdate function Send-TextMessage { <# .SYNOPSIS Remove illegal characters from exceptions, so they can be posted to operators #> param ($message, $uri) try { $text = $message -replace "[`r`n']+", ' ' $TeamsMessageContent = New-Object PSObject -Property @{text = $text } | ConvertTo-Json Write-CustomLog $message foreach ($connector in $uri) { Invoke-RestMethod -Uri $connector -Method Post -Body $TeamsMessageContent -ContentType 'application/json' } } catch { throw ("Send-TextMessage failed with: {0}" -f $_.exception.message) } } function Format-OperatorMessage { <# .SYNOPSIS Add emphasis to certain messages to draw operator's attention #> param ($message) try { $strongmsgs = @('Collecting Azure Stack Logs', 'We hit a problem sending data from the stamp to teams', 'URP return unknown state', 'PreparationFailed', 'InstallationFailed', 'Update is installing but PEP Session lost', 'Update AzS (Update|Hotfix)', 'Oem Package Update', 'Update (.*) is') $message = $message | Foreach-Object { if ($PSITEM -match ($strongmsgs -join '|')) { "<strong>$PSITEM</strong>" } else { $PSITEM } } return (New-Object PSObject -Property @{text = '<body><BR>' + ($message -join '<BR>') } | ConvertTo-Json) } catch { throw ("Format-OperatorMessage failed with {0}" -f $_.exception.message) } } function Invoke-PepCommand { <# .SYNOPSIS Try commands and reestablish PEP connection incase of breakage. #> param ($pepsession,$scriptblock, $retrySeconds = 30,[switch]$AsJob) $retries = 1 while ($retries -le 3) { try { $result = Invoke-Command -session $pepsession -scriptblock $scriptblock -AsJob:$AsJob return $result } catch { Write-CustomLog ("PEP Command ($retries/3) retries failed with : {0}" -f $_.exception.message) $pepsession = Get-PepSessionWithRetries -stamp $stamp -retrySeconds $retrySeconds -PepCredential $PepCredential $retries++ Start-Sleep -seconds $retrySeconds } } } # SIG # Begin signature block # MIIjkgYJKoZIhvcNAQcCoIIjgzCCI38CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCBzlJk/ghVkVSkG # 6gddAVZI9GRCa/jIMl9bIl1FOcHhU6CCDYEwggX/MIID56ADAgECAhMzAAABh3IX # chVZQMcJAAAAAAGHMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjAwMzA0MTgzOTQ3WhcNMjEwMzAzMTgzOTQ3WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDOt8kLc7P3T7MKIhouYHewMFmnq8Ayu7FOhZCQabVwBp2VS4WyB2Qe4TQBT8aB # znANDEPjHKNdPT8Xz5cNali6XHefS8i/WXtF0vSsP8NEv6mBHuA2p1fw2wB/F0dH # sJ3GfZ5c0sPJjklsiYqPw59xJ54kM91IOgiO2OUzjNAljPibjCWfH7UzQ1TPHc4d # weils8GEIrbBRb7IWwiObL12jWT4Yh71NQgvJ9Fn6+UhD9x2uk3dLj84vwt1NuFQ # itKJxIV0fVsRNR3abQVOLqpDugbr0SzNL6o8xzOHL5OXiGGwg6ekiXA1/2XXY7yV # Fc39tledDtZjSjNbex1zzwSXAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUhov4ZyO96axkJdMjpzu2zVXOJcsw # UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1 # ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDU4Mzg1MB8GA1UdIwQYMBaAFEhu # ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu # bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w # Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx # MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAixmy # S6E6vprWD9KFNIB9G5zyMuIjZAOuUJ1EK/Vlg6Fb3ZHXjjUwATKIcXbFuFC6Wr4K # NrU4DY/sBVqmab5AC/je3bpUpjtxpEyqUqtPc30wEg/rO9vmKmqKoLPT37svc2NV # BmGNl+85qO4fV/w7Cx7J0Bbqk19KcRNdjt6eKoTnTPHBHlVHQIHZpMxacbFOAkJr # qAVkYZdz7ikNXTxV+GRb36tC4ByMNxE2DF7vFdvaiZP0CVZ5ByJ2gAhXMdK9+usx # zVk913qKde1OAuWdv+rndqkAIm8fUlRnr4saSCg7cIbUwCCf116wUJ7EuJDg0vHe # yhnCeHnBbyH3RZkHEi2ofmfgnFISJZDdMAeVZGVOh20Jp50XBzqokpPzeZ6zc1/g # yILNyiVgE+RPkjnUQshd1f1PMgn3tns2Cz7bJiVUaqEO3n9qRFgy5JuLae6UweGf # AeOo3dgLZxikKzYs3hDMaEtJq8IP71cX7QXe6lnMmXU/Hdfz2p897Zd+kU+vZvKI # 3cwLfuVQgK2RZ2z+Kc3K3dRPz2rXycK5XCuRZmvGab/WbrZiC7wJQapgBodltMI5 # GMdFrBg9IeF7/rP4EqVQXeKtevTlZXjpuNhhjuR+2DMt/dWufjXpiW91bo3aH6Ea # jOALXmoxgltCp1K7hrS6gmsvj94cLRf50QQ4U8Qwggd6MIIFYqADAgECAgphDpDS # AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 # ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla # MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT # H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG # OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S # 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz # y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7 # 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u # M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33 # X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl # XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP # 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB # l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF # RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM # CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ # BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud # DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO # 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0 # LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB # FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw # cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA # XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY # 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj # 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd # d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ # Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf # wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ # aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j # NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B # xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96 # eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7 # r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I # RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIVZzCCFWMCAQEwgZUwfjELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z # b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAYdyF3IVWUDHCQAAAAABhzAN # BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor # BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQghp2lRLsg # jNZY8VFfRyz5848xy6mrKuuM5UIsXH4DO1IwQgYKKwYBBAGCNwIBDDE0MDKgFIAS # AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN # BgkqhkiG9w0BAQEFAASCAQAwqZ067w53D2G7/+lxRQQjHOQrPmspGjz3GS4z2Az1 # fIYYW2JX6qV8Zk2zYfidbLE3EY8fcVtUPi0qJ3j+ly1WK39c8kpTLyDw1p10kaJj # iKjgb+dlKkAl5eGlgJsPjea2PN7tQHMYATIF9ElW0A26w20ve1HQVhFgV8I5HTCN # taO9qPuVCUZuKIPSPfqde3Ihnj0w3zAahKbIy5Xid3VlRPu/Yo0j+noVnEYKrNZX # A/g1XcGkGJ3dGKE10L964sJbU/8kGTeXaYUrkyXUcrd0Y1TySkIBFsMW9rGjKUsA # s8UNVHlKgRJ9DkQBvCjJHpf5qWxJj6tvj+72Be/WTZnJoYIS8TCCEu0GCisGAQQB # gjcDAwExghLdMIIS2QYJKoZIhvcNAQcCoIISyjCCEsYCAQMxDzANBglghkgBZQME # AgEFADCCAVUGCyqGSIb3DQEJEAEEoIIBRASCAUAwggE8AgEBBgorBgEEAYRZCgMB # MDEwDQYJYIZIAWUDBAIBBQAEIKSHsIADZIw2sjaVvNjVqK1TefBzHCZEQVjbOyVO # f3VKAgZfiERj7BAYEzIwMjAxMTEyMjEyODU4LjA4N1owBIACAfSggdSkgdEwgc4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKTAnBgNVBAsTIE1p # Y3Jvc29mdCBPcGVyYXRpb25zIFB1ZXJ0byBSaWNvMSYwJAYDVQQLEx1UaGFsZXMg # VFNTIEVTTjo3ODgwLUUzOTAtODAxNDElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUt # U3RhbXAgU2VydmljZaCCDkQwggT1MIID3aADAgECAhMzAAABKKAOgeE21U/CAAAA # AAEoMA0GCSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNo # aW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29y # cG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEw # MB4XDTE5MTIxOTAxMTUwMFoXDTIxMDMxNzAxMTUwMFowgc4xCzAJBgNVBAYTAlVT # MRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQK # ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVy # YXRpb25zIFB1ZXJ0byBSaWNvMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo3ODgw # LUUzOTAtODAxNDElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vydmlj # ZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ2Rsdb3VNuGPs2/Dgpc # 9gt77LG0JPkD4VWTlEJLkqznTJl+RoZfiOwN6iWfPu4k/kj8nwY7pvLs1OsBy494 # yusg4rHLwHNUJPtw1Tc54MOLgdcosA4Nxki73fDyqWwDtjOdk6H7kNczBPqADD6B # 98ot77/wSACBJIxm9qAUudquS5fczCF0++aWUavDu46U3cv6HEjIdV2ZdJTUKg4W # UIdTYMQXI082+qSs45WBZjcK98/tIfx8uq8q8ksWF9+zUjGTFiMaKHhn7cSCoEj7 # E1tVmW08ISpS678WFP2+A0OQwaWcJKNACK+J+La7Lz2bGupCidOGz5XDewc1lD9n # LPcCAwEAAaOCARswggEXMB0GA1UdDgQWBBSE4vKD8X61N5vUAcNOdH9QBMum8jAf # BgNVHSMEGDAWgBTVYzpcijGQ80N7fEYbxTNoWoVtVTBWBgNVHR8ETzBNMEugSaBH # hkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNU # aW1TdGFQQ0FfMjAxMC0wNy0wMS5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUF # BzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1RpbVN0 # YVBDQV8yMDEwLTA3LTAxLmNydDAMBgNVHRMBAf8EAjAAMBMGA1UdJQQMMAoGCCsG # AQUFBwMIMA0GCSqGSIb3DQEBCwUAA4IBAQCLX2ZHGIULgDk/iccHWUywjDyAsBHl # hkmtmBp4lldwL3dNo0bXZZHiSZB+c2KzvPqY64BlECjS/Pqur2m9UaT1N0BeUowR # HQT88wdzd94gYqKXmLDbVR8yeVgBkcP/JiVWbXdQzcz1ETHgWrh+uzA8BwUgAaHJ # w+nXYccIuDgPJM1UTeNl9R5Ovf+6zR2E5ZI4DrIqvS4jH4QsoMPTn27AjN7VZt4a # moRxMLEcQAS7vPT1JUUaRFpFHmkUYVln1YMsw///6968aRvy3cmClS44uxkkaILb # hh1h09ejZjHhrEn+k9McVkWiuY724jJ/57tylM7A/jzIWNj1F8VlhkyyMIIGcTCC # BFmgAwIBAgIKYQmBKgAAAAAAAjANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMC # VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV # BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJv # b3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTAwHhcNMTAwNzAxMjEzNjU1WhcN # MjUwNzAxMjE0NjU1WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3Rv # bjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0 # aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCASIw # DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKkdDbx3EYo6IOz8E5f1+n9plGt0 # VBDVpQoAgoX77XxoSyxfxcPlYcJ2tz5mK1vwFVMnBDEfQRsalR3OCROOfGEwWbEw # RA/xYIiEVEMM1024OAizQt2TrNZzMFcmgqNFDdDq9UeBzb8kYDJYYEbyWEeGMoQe # dGFnkV+BVLHPk0ySwcSmXdFhE24oxhr5hoC732H8RsEnHSRnEnIaIYqvS2SJUGKx # Xf13Hz3wV3WsvYpCTUBR0Q+cBj5nf/VmwAOWRH7v0Ev9buWayrGo8noqCjHw2k4G # kbaICDXoeByw6ZnNPOcvRLqn9NxkvaQBwSAJk3jN/LzAyURdXhacAQVPIk0CAwEA # AaOCAeYwggHiMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBTVYzpcijGQ80N7 # fEYbxTNoWoVtVTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMC # AYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvX # zpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20v # cGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYI # KwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNydDCBoAYDVR0g # AQH/BIGVMIGSMIGPBgkrBgEEAYI3LgMwgYEwPQYIKwYBBQUHAgEWMWh0dHA6Ly93 # d3cubWljcm9zb2Z0LmNvbS9QS0kvZG9jcy9DUFMvZGVmYXVsdC5odG0wQAYIKwYB # BQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AUABvAGwAaQBjAHkAXwBTAHQAYQB0AGUA # bQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAAfmiFEN4sbgmD+BcQM9naOh # IW+z66bM9TG+zwXiqf76V20ZMLPCxWbJat/15/B4vceoniXj+bzta1RXCCtRgkQS # +7lTjMz0YBKKdsxAQEGb3FwX/1z5Xhc1mCRWS3TvQhDIr79/xn/yN31aPxzymXlK # kVIArzgPF/UveYFl2am1a+THzvbKegBvSzBEJCI8z+0DpZaPWSm8tv0E4XCfMkon # /VWvL/625Y4zu2JfmttXQOnxzplmkIz/amJ/3cVKC5Em4jnsGUpxY517IW3DnKOi # PPp/fZZqkHimbdLhnPkd/DjYlPTGpQqWhqS9nhquBEKDuLWAmyI4ILUl5WTs9/S/ # fmNZJQ96LjlXdqJxqgaKD4kWumGnEcua2A5HmoDF0M2n0O99g/DhO3EJ3110mCII # YdqwUB5vvfHhAN/nMQekkzr3ZUd46PioSKv33nJ+YWtvd6mBy6cJrDm77MbL2IK0 # cs0d9LiFAR6A+xuJKlQ5slvayA1VmXqHczsI5pgt6o3gMy4SKfXAL1QnIffIrE7a # KLixqduWsqdCosnPGUFN4Ib5KpqjEWYw07t0MkvfY3v1mYovG8chr1m1rtxEPJdQ # cdeh0sVV42neV8HR3jDA/czmTfsNv11P6Z0eGTgvvM9YBS7vDaBQNdrvCScc1bN+ # NR4Iuto229Nfj950iEkSoYIC0jCCAjsCAQEwgfyhgdSkgdEwgc4xCzAJBgNVBAYT # AlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYD # VQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKTAnBgNVBAsTIE1pY3Jvc29mdCBP # cGVyYXRpb25zIFB1ZXJ0byBSaWNvMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo3 # ODgwLUUzOTAtODAxNDElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy # dmljZaIjCgEBMAcGBSsOAwIaAxUAMT1LG/KAEj0XsiL9n7mxmX1afZuggYMwgYCk # fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF # AONXq2owIhgPMjAyMDExMTIxNjM5MDZaGA8yMDIwMTExMzE2MzkwNlowdzA9Bgor # BgEEAYRZCgQBMS8wLTAKAgUA41eragIBADAKAgEAAgISngIB/zAHAgEAAgISlzAK # AgUA41j86gIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIB # AAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBAG/gHnCcf6zUuLxg # ax/g78hDO29QFce4ULZPsDAZlT6bGOBVQPzfxY/9yKbaSQtAHJbExtoZQjnh3IAj # AF3B33txbM/xtqG7Ji2ww2PzG/+isv1MRUHlUVrZipzHrQG96HTu3D61HKmr/Bnm # /5t06PbWm4CRTmoBH4Tlrb8XhuSNMYIDDTCCAwkCAQEwgZMwfDELMAkGA1UEBhMC # VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV # BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp # bWUtU3RhbXAgUENBIDIwMTACEzMAAAEooA6B4TbVT8IAAAAAASgwDQYJYIZIAWUD # BAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0B # CQQxIgQgtv00IVUsKA8DxU0YE9G5IlIq7LWv0sBi25F/rqg5kDQwgfoGCyqGSIb3 # DQEJEAIvMYHqMIHnMIHkMIG9BCC8RWqLrwVSd+/cGxDfBqS4b1tPXhoPFrC615vV # 1ugU2jCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u # MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp # b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB # KKAOgeE21U/CAAAAAAEoMCIEIHFreWgEKx9iW+szJ1TwSbRhfpBmlVHvslavewhT # aks0MA0GCSqGSIb3DQEBCwUABIIBADVLLQch76mTY+d8hWE14xBeo7bkIPeKNHTt # UokPj3SOrhOos+/dVSAEC/puZH6z9M09Vc7wO+BAUFi8ReEQZ+G496TpqTCZ+HXJ # B6p8a0MeG3wujrCP/4HTSFmwLIQ3gZFBjRRnL3Frzc0q2MCzFULL5voOUZ7IbpG7 # ky/S1+lysuz+lXkARpDjLoIswU9v3cFKOCk5lWJzInOyP5luIr4X+JRkVOyDXZWN # 0GW1U8ImlXi9BueFnI42pKXgiZBhkAjr8oNTrJe1AUTcuH4y/bNvFiPTm5s1mIxD # Pgt6gf8lV/65eE9yNORlCiQGaZCwSMfwfgsjdlkFWw6B/UgRXwI= # SIG # End signature block |