Obs/bin/ObsDep/content/Powershell/Common/BCDRActionHelper.psm1
|
<###################################################
# # # Copyright (c) Microsoft. All rights reserved. # # # ##################################################> Import-Module $PSScriptRoot\Tracer.psm1 $BackupStartedEventLogPattern = "Action {0} started with session name <{1}>, session shell ID {2} and process ID {3}." $BackupCompletedEventLogPattern = "Action {0} completed with session name <{1}>." $BackupFailedEventLogPattern = "Action {0} failed with session name <{1}>. Exception: {2}" $BCDRSessionNameBase = "AzsBCDRSession" $BCDRLogName = "AzSBCDR" $BCDRBackupEventSource = "AzSBackupAction" # TODO: restore actions are not utilizing the action helper at the moment. $BCDRRestoreEventSource = "AzSRestoreAction" $BCDRRemoteActionStartedEventId = 1 $BCDRRemoteActionCompletededEventId = 2 $BCDRRemoteActionFailedEventId = 3 # 4 hour session idle timeout $BCDRRemoteSessionIdleTimeoutInSec = 14400 <# .Synopsis Get the BCDR action plan constants #> function Get-BCDRActionPlanConsts { $consts = @{} $consts.BackupStartedEventLogPattern = $BackupStartedEventLogPattern $consts.BackupCompletedEventLogPattern = $BackupCompletedEventLogPattern $consts.BackupFailedEventLogPattern = $BackupFailedEventLogPattern $consts.BCDRSessionNameBase = $BCDRSessionNameBase $consts.BCDRLogName = $BCDRLogName $consts.BCDRBackupEventSource = $BCDRBackupEventSource $consts.BCDRRestoreEventSource = $BCDRRestoreEventSource $consts.BCDRRemoteActionStartedEventId = $BCDRRemoteActionStartedEventId $consts.BCDRRemoteActionCompletededEventId = $BCDRRemoteActionCompletededEventId $consts.BCDRRemoteActionFailedEventId = $BCDRRemoteActionFailedEventId return $consts } <# .Synopsis Get the BCDR action plan constant #> function Get-BCDRActionPlanConst { [CmdletBinding()] Param ( [Parameter(Mandatory = $true)] [ValidateSet('BackupStartedEventLogPattern', 'BackupCompletedEventLogPattern', 'BackupFailedEventLogPattern', ` 'BCDRSessionNameBase', 'BCDRLogName', 'BCDRBackupEventSource', 'BCDRRestoreEventSource', ` 'BCDRRemoteActionStartedEventId', 'BCDRRemoteActionCompletededEventId', 'BCDRRemoteActionFailedEventId')] [string] $ConstName ) $consts = Get-BCDRActionPlanConsts return $consts[$ConstName] } <# .Synopsis Create event source and log on target VMs #> function Ensure-BCDREventLog { param ( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $ComputerName, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [PSCredential] $Credential, [Parameter(Mandatory=$true)] [string] $EventSource ) Invoke-Command -ComputerName $ComputerName -Credential $Credential -ArgumentList @($BCDRLogName, $EventSource) ` -ScriptBlock { param ($BCDRLogName, $EventSource) if ([System.Diagnostics.EventLog]::SourceExists($EventSource) -eq $false) { Trace-Execution "Creating event source $EventSource on event log $BCDRLogName" [System.Diagnostics.EventLog]::CreateEventSource($EventSource, $BCDRLogName) Trace-Execution "Event source $EventSource created" Limit-EventLog -LogName $BCDRLogName -RetentionDays 30 } } } <# .Synopsis Get BCDR action event logs with a specific backup from the target VM #> function Get-BCDRActionEventLog { param ( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $ComputerName, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [PSCredential] $Credential, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $BackupId, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $SessionName ) return Invoke-Command -ComputerName $ComputerName -Credential $Credential -ArgumentList @($BackupId, $BCDRLogName, $SessionName) -ScriptBlock { param ($BackupId, $BCDRLogName, $SessionName) try { $logs = Get-EventLog -LogName $BCDRLogName -ErrorAction Stop ` | ? { ($_.Message -like "*$BackupId*") -and ($_.Message -like "*<$SessionName>*") } # Log the last 5 logs about this backup session for diagnostics $numHistory = 5 foreach ($log in ($logs | select -First $numHistory)) { Trace-Execution "Last $numHistory logs for backup $BackupId and session '$SessionName'" Trace-Execution "$($log.TimeGenerated), EventId: $($log.EventID), Msg: $($log.Message)" } return $logs } catch {} } } <# .Synopsis Check if the BCDR remote session process still exists and kill the process if requested to #> function Check-BCDRSessionProcess { param ( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $ComputerName, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [PSCredential] $Credential, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $SessionPid, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [DateTime] $SessionCreationTime, [switch] $Kill ) $proc = $null $retries = 10 while (!$proc) { try { # Use Invoke-Command instead of 'Get-Process -CompueterName' to make sure process StartTime properties can # be returned $proc = Invoke-Command -ComputerName $ComputerName -Credential $Credential -ScriptBlock { Get-Process -ProcessName 'wsmprovhost' -ErrorAction Stop } } catch { $retries-- if ($retries -gt 0) { Trace-Execution "Failed to get PowerShell remote host processes on $ComputerName. Exception: $_. Retrying..." Start-Sleep -s 10 continue } throw } } $sessionProc = $proc | ? { ($_.Id -eq $SessionPid)} if ($sessionProc) { # Check the creation time to prevent accidental termination of other processes with the same Pid. # Session process must be started earlier than session log time. Give it 1 extra second to the session creation # time because ETL log time has seconds precision while the process start time has milliseconds precision. $sessionProcStartTimeUTC = $sessionProc.StartTime.ToUniversalTime() $sessionCreationTimeUTC = $SessionCreationTime.ToUniversalTime().AddSeconds(1) if ($sessionProcStartTimeUTC -le $sessionCreationTimeUTC) { if ($Kill.IsPresent) { Invoke-Command -ComputerName $ComputerName -Credential $Credential -ArgumentList @($SessionPid) ` -ScriptBlock { param ($SessionPid) Stop-Process -Id $SessionPid -Force -ErrorAction SilentlyContinue } } return $true } else { Trace-Execution "The process with Id $SessionPid isn't the remote PowerShell host process for this invocation." Trace-Execution "Session process start time: $sessionProcStartTimeUTC, Session creation time: $sessionCreationTimeUTC" } } else { $str = $null $proc | select Id, ProcessName | % { $str += "($($_.Id), $($_.ProcessName))`n" } Trace-Execution "All PowerShell remote host processes names on $ComputerName :`n$str" Trace-Execution "Failed to find a remote PowerShell host process with process Id $SessionPid on $ComputerName." } return $false } <# .Synopsis Check previously started BCDR remote sessions. The BCDR remote sessions for each role (and each invocation if a role has more than one) are named with this pattern: '<BCDRSessionNameBase>-<FullRepositoryName>' This method searches the remote session name on all nodes for the current role, try to get the action status to determine next actions, and finally clean up the sessions. #> function Check-PreviousBCDRSession { param ( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string[]] $NodesForCurrentRole, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [PSCredential] $Credential, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $BackupId, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $SessionName, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $SessionInfoEventRegex, [Parameter(Mandatory=$false)] $ProvisionedPSSession = $null, [Parameter(Mandatory=$false)] [PSCredential] $ProvisionedPSSessionCredential = $null, [Parameter(Mandatory=$true)] [uint32] $ExpectedRuntimeInMin, [Parameter(Mandatory=$true)] [ref] $ShouldInvokeAction ) $ShouldInvokeAction.Value = $true # Test if the session already exists on all the nodes $sessions = @() Trace-Execution "Searching for existing PSSessions on $NodesForCurrentRole" foreach ($Node in $NodesForCurrentRole) { if ($ProvisionedPSSessionCredential) { $sessions += Get-PSSession -ComputerName $Node -Credential $ProvisionedPSSessionCredential -Name $SessionName -ErrorAction SilentlyContinue } else { $sessions += Get-PSSession -ComputerName $Node -Credential $Credential -Name $SessionName -ErrorAction SilentlyContinue } } # Make sure to filter out the provisioned PSSession if specified. Trace-Execution "Found sessions: $sessions" if ($ProvisionedPSSession) { $sessions = $sessions | ? { if ($ProvisionedPSSession.InstanceId -ne $_.InstanceId) { return $true } else { Trace-Execution "Filter out the provisioned PSSession with ID $($ProvisionedPSSession.InstanceId) from the sessions." return $false } } } foreach ($session in $sessions) { # Properly working BCDR remote sessions should be in the 'Disconnected' state and ready to be connected try { Trace-Execution "The disconnected remote PSSession '$($session.Name)' exists on $($session.ComputerName)." Trace-Execution "State: $($session.State), Availability: $($session.Availability), Instance ID: $($session.InstanceId)." Trace-Execution "Check the previous backup status of BackupID $backupID." # Get the latest log with this backup ID $latestLog = Get-BCDRActionEventLog -ComputerName $session.ComputerName -Credential $Credential ` -BackupId $backupId -SessionName $SessionName | select -First 1 if ($latestLog) { switch ($latestLog.EventId) { # Started $BCDRRemoteActionStartedEventId { $expectedCompletion = $latestLog.TimeGenerated.ToUniversalTime().AddMinutes($ExpectedRuntimeInMin) Trace-Execution "Action started at UTC $($latestLog.TimeGenerated.ToUniversalTime().ToString())" Trace-Execution "Expected to complete by UTC $($expectedCompletion.ToUniversalTime().ToString())" $actionPlanDone = $false do { # Check for completion $now = Get-Date Trace-Execution "Current time is UTC $($now.ToUniversalTime().ToString())" $log = Get-BCDRActionEventLog -ComputerName $session.ComputerName -Credential $Credential ` -BackupId $backupId -SessionName $SessionName | select -First 1 if ($log.EventId -eq $BCDRRemoteActionCompletededEventId) { $ShouldInvokeAction.Value = $false $actionPlanDone = $true } elseif ($log.EventId -eq $BCDRRemoteActionFailedEventId) { Trace-Execution "The previous action failed, invoking the action again" $actionPlanDone = $true } if ($actionPlanDone) { break } # Keep waiting Start-Sleep -Seconds 30 } while ($now -le $expectedCompletion) if (!$actionPlanDone) { Trace-Execution "The previous action plan didn't finish in time, restarting another one." } } # Completed $BCDRRemoteActionCompletededEventId { $shouldInvokeAction.Value = $false } # Failed $BCDRRemoteActionFailedEventId { Trace-Execution "The previous action failed, invoking the action again" } } } break } catch {} } if ($sessions) { Trace-Execution "Removing the disconnected PSSessions." $sessions | Remove-PSSession -ErrorAction SilentlyContinue } foreach ($Node in $NodesForCurrentRole) { try { Trace-Execution "Make sure the WSMan instance and the process are both gone on $Node" $instances = Get-WSManInstance -ConnectionURI "http://$Node`:5985/wsman" shell -Enumerate ` -cred $Credential -ErrorAction SilentlyContinue | ? Name -eq $SessionName if ($ProvisionedPSSession) { $instances = $instances | ? { if ($ProvisionedPSSession.InstanceId -ne $_.ShellId) { return $true } else { Trace-Execution "Filter out the provisioned PSSession ID $($ProvisionedPSSession.InstanceId) from the instances" return $false } } } foreach ($instance in $instances) { Trace-Execution "Try to remove the WSMan object with shell ID $($instance.ShellId)" Remove-WSManInstance -ConnectionURI "http://$Node`:5985/wsman" shell @{ShellID="$($instance.ShellId)"} ` -cred $Credential -ErrorAction SilentlyContinue } $logs = Get-BCDRActionEventLog -ComputerName $Node -Credential $Credential -BackupId $backupId ` -SessionName $SessionName | ? EventID -eq $BCDRRemoteActionStartedEventId if ($logs) { Trace-Execution "Check the previous session PID." foreach ($log in $logs) { Trace-Execution "Log message: $($log.Message)" if ($log.Message -Match $SessionInfoEventRegex) { $sessionNameFromLog = $Matches[2] if ($sessionNameFromLog -ne $SessionName) { Trace-Execution "Not for the current session, skipped." continue } $wsPid = $Matches[4] $logTime = $log.TimeGenerated.ToUniversalTime() if (![string]::IsNullOrEmpty($wsPid)) { Trace-Execution "Make sure process '$wsPid' is gone." $null = Check-BCDRSessionProcess -ComputerName $Node -Credential $Credential ` -SessionPid $wsPid -SessionCreationTime $logTime -Kill } } } } } catch { Trace-Execution "Failed to clean up the WSMan instance and/or the process on $Node. Proceeding anyway. Exception: $_" } } } <# .Synopsis A wrapper around actual execution on remote machines when the execution runs in a JEA session, in which case this method must be whitelisted in the session configuration (.psrc) or imported manually as part of the initialization #> function Start-BCDRExecutionOnRemoteComputer { param ( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $BCDRRemoteExecutionScriptBlockAsStr, [Parameter(Mandatory=$true)] $ScriptBlockParameters ) $BCDRRemoteExecutionScriptBlock = [scriptblock]::Create($BCDRRemoteExecutionScriptBlockAsStr) # Start the script block execution async Start-Job -ScriptBlock $BCDRRemoteExecutionScriptBlock -ArgumentList $ScriptBlockParameters } <# .Synopsis Execute backup operations idempotently. BCDR ECE actions start remote PSSession on VMs for roles that implements backup/restore interfaces to perform backup/restore actions. This method ensures that these sessions can be tracked or cleanly canceled when ECE fails over and/or crashes by disconnecting the worker PSSessions and poll for completion. .PARAMETER ComputerName The targe computer name where the execution is run. If ProvisionedPSSession isn't specified, a disconnected session will be created for the caller. Otherwise the computer name is used to poll action plan results. .PARAMETER Credential The credential to access the computer. .PARAMETER BackupID Backup ID .PARAMETER FullRepositoryName The full repository name is used to construct the PSSession name, which should be distinct for each repository so that they can be tracked individually. The pattern is 'AzsBCDRSession-<FullRepostiroyName>' .PARAMETER NodesForCurrentRole List of Nodes for the role where the method should look for previously started BCDR remote PSSessions .PARAMETER EmbeddedScriptBlockAsStr The script block that contains the codes for the actual execution on the remote machine, passed in as a string. The author of the script block must ensure that the codes runs in constrained language mode, otherwise the author must manually set full language mode in this script block. .PARAMETER ScriptBlockParameters The parameter list of the embedded script block .PARAMETER ProvisionedPSSession The provisioned PSSession. The session name must follow this pattern: 'AzsBCDRSession-<FullRepositoryName>'. Once the codes in the EmbeddedScriptBlockAsStr is executed as a job in the session, the session will be disconnected and the caller should *NOT* try to reconnect to it if a reference is still held. .PARAMETER ExpectedRuntimeInMin The method first checks if there were any previously started PSSessions, and if there is any, it finds the start time of the execution and poll for results until the run exceeds the expected run time. .PARAMETER ConfigurationName The configuration name to start the disconnected session with. If not specified, default to use CredSSP .PARAMETER ActionType Specifies whether the action is a backup or restore #> function Start-IdempotentBCDRRemoteExecution { param ( [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)] [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $ComputerName, [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)] [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)] [ValidateNotNullOrEmpty()] [PSCredential] $Credential, [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)] [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $BackupID, [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)] [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$false)] [ValidateNotNullOrEmpty()] [string] $FullRepositoryName, [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)] [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)] [ValidateNotNullOrEmpty()] [string[]] $NodesForCurrentRole, [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)] [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $EmbeddedScriptBlockAsStr, [Parameter(Mandatory=$false)] [Array] $ScriptBlockParameters, [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)] [ValidateNotNullOrEmpty()] $ProvisionedPSSession = $null, [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)] [ValidateNotNull()] [PSCredential] $ProvisionedPSSessionCredential = $null, [Parameter(Mandatory=$false)] [uint32] $ExpectedRuntimeInMin = 60 * 4, [Parameter(ParameterSetName = "DefaultSet", Mandatory=$false)] [string] $ConfigurationName = $null, [Parameter(Mandatory = $false)] [ValidateSet('Backup', 'Restore')] [string] $ActionType = 'Backup', [Parameter(Mandatory = $false)] [string] $BackupSnapshotFolder, [Parameter(Mandatory = $false)] [string[]] $BackupSnapshotFilters ) try { $defaultSessionName = "$BCDRSessionNameBase-$FullRepositoryName" if ($ProvisionedPSSession) { if ($ProvisionedPSSession.Name -notlike "$defaultSessionName*") { throw "Unexpected PSSession name '$($ProvisionedPSSession.Name)'. The session must start with '$defaultSessionName'." } $sessionName = $ProvisionedPSSession.Name } else { $sessionName = $defaultSessionName } if ($ActionType -eq 'Backup') { $EventSource = $BCDRBackupEventSource } else { $EventSource = $BCDRRestoreEventSource } $sessionInfoEventRegex = $BackupStartedEventLogPattern -f "([\d\w\-]+)", "(.+)", "([\d\w\-]+)", "(\d+)" $shouldInvokeAction = $true Check-PreviousBCDRSession -NodesForCurrentRole $NodesForCurrentRole -Credential $Credential -BackupID $BackupID ` -SessionName $sessionName -SessionInfoEventRegex $sessionInfoEventRegex ` -ProvisionedPSSession $ProvisionedPSSession -ProvisionedPSSessionCredential $ProvisionedPSSessionCredential ` -ExpectedRuntimeInMin $ExpectedRuntimeInMin -ShouldInvokeAction ([ref] $shouldInvokeAction) if ($shouldInvokeAction -eq $false) { Trace-Execution "The previous action is done, skip invoking the action again" return } Trace-Execution "Ensure the BCDR event source and log exists" Ensure-BCDREventLog -ComputerName $ComputerName -Credential $Credential -EventSource $EventSource $BCDREventParams = @{} $BCDREventParams.BCDRLogName = $BCDRLogName $BCDREventParams.EventSource = $EventSource $BCDREventParams.BCDRRemoteActionStartedEventId = $BCDRRemoteActionStartedEventId $BCDREventParams.BCDRRemoteActionCompletededEventId = $BCDRRemoteActionCompletededEventId $BCDREventParams.BCDRRemoteActionFailedEventId = $BCDRRemoteActionFailedEventId $BCDREventParams.BackupStartedEventLogPattern = $BackupStartedEventLogPattern $BCDREventParams.BackupCompletedEventLogPattern = $BackupCompletedEventLogPattern $BCDREventParams.BackupFailedEventLogPattern = $BackupFailedEventLogPattern $BCDRRemoteExecutionScriptBlockParams = @($BCDREventParams, $sessionName, $BackupID, ` $EmbeddedScriptBlockAsStr, $ScriptBlockParameters, $BackupSnapshotFolder, $BackupSnapshotFilters) $BCDRRemoteExecutionScriptBlock = { param ($BCDREventParams, $sessionName, $BackupID, $EmbeddedScriptBlockAsStr, $ScriptBlockParameters, $BackupSnapshotFolder, $BackupSnapshotFilters) try { Import-Module OpenUpSession Set-FullLanguage } catch{} # Log the shell ID and the PID of the disconnected session $instance = Get-WSManInstance -ConnectionURI "http://localhost:5985/wsman" shell -Enumerate ` | Where-Object Name -eq $sessionName $backupStartedEventLogMsg = $BCDREventParams.BackupStartedEventLogPattern -f $BackupID, $sessionName, ` $instance.ShellId, $instance.ProcessId Write-EventLog -LogName $BCDREventParams.BCDRLogName -Source $BCDREventParams.EventSource ` -EntryType Information -EventId $BCDREventParams.BCDRRemoteActionStartedEventId -Message $backupStartedEventLogMsg $sb = [scriptblock]::Create($EmbeddedScriptBlockAsStr) $succeeded = $false try { Invoke-Command -ArgumentList $ScriptBlockParameters -ScriptBlock $sb -ErrorAction Stop $succeeded = $true } catch { $errorRecord = $_ } finally { $errorMsg = $errorRecord | select * | Out-String # Double check the snapshots to adjust the final result if ($BackupSnapshotFolder -and ($BackupSnapshotFilters.Count -gt 0)) { try { $logFileFullPath = Join-Path "$env:SystemDrive\MASLogs" -ChildPath "$($sessionName)_$($BackupID)_$(Get-Date -Format "yyyyMMdd-HHmmss").log" Start-Transcript -Append -Path $logFileFullPath Write-Verbose "Check snapshots under $BackupSnapshotFolder with $BackupSnapshotFilters finally to adjust result" $snapshotNotFound = $false foreach ($filter in $BackupSnapshotFilters) { $snapshots = Get-ChildItem $BackupSnapshotFolder -Filter $filter -Force if (!$snapshots) { Write-Verbose "Failed to find snapshot with $filter" $succeeded = $false $snapshotNotFound = $true $errorMsg += "`nNo snapshot found under $BackupSnapshotFolder with filter $filter." } } if (!$snapshotNotFound) { Write-Verbose "Succeeded to find all snapshots, adjust result from $succeeded to true" $succeeded = $true } } catch { Write-Verbose "Ignore exception $($_.Exception.Message) when checking snapshots" } finally { Stop-Transcript -ErrorAction Ignore } } if ($succeeded) { $backupCompletedEventLogMsg = $BCDREventParams.BackupCompletedEventLogPattern -f $BackupID, $sessionName Write-EventLog -LogName $BCDREventParams.BCDRLogName -Source $BCDREventParams.EventSource ` -EntryType Information -EventId $BCDREventParams.BCDRRemoteActionCompletededEventId ` -Message $backupCompletedEventLogMsg } else { $backupFailedEventLogMsg = $BCDREventParams.BackupFailedEventLogPattern -f $BackupID, $sessionName, $errorMsg Write-EventLog -LogName $BCDREventParams.BCDRLogName -Source $BCDREventParams.EventSource ` -EntryType Error -EventId $BCDREventParams.BCDRRemoteActionFailedEventId ` -Message $backupFailedEventLogMsg throw $errorMsg, $errorRecord } } } if ($PsCmdlet.ParameterSetName -eq 'ProvisionedPSSessionSet') { Trace-Execution "Using the provisioned PSSession." Trace-Execution "Name: $($ProvisionedPSSession.Name), State: $($ProvisionedPSSession.State)" Trace-Execution "Availability: $($ProvisionedPSSession.Availability), ConfigName: $($ProvisionedPSSession.ConfigurationName)" # Execute the script block in the provisioned session and disconnect # # Most cases where the caller passes in a provisioned PSSession is because the session was created with a # JEA endpoint and specific language mode is enforced. Therefore, Start-BCDRExecutionOnRemoteComputer must # be imported in the provisioned PSSession already and cannot be imported here to support this scenario. Invoke-Command -Session $ProvisionedPSSession ` -ArgumentList @($BCDRRemoteExecutionScriptBlock.ToString(), $BCDRRemoteExecutionScriptBlockParams) ` -ScriptBlock { param ($BCDRRemoteExecutionScriptBlockAsStr, $BCDRRemoteExecutionScriptBlockParams) Start-BCDRExecutionOnRemoteComputer ` -BCDRRemoteExecutionScriptBlockAsStr $BCDRRemoteExecutionScriptBlockAsStr ` -ScriptBlockParameters $BCDRRemoteExecutionScriptBlockParams } Trace-Execution "Disconnecting the remote PSSession." Disconnect-PSSession -Session $ProvisionedPSSession -IdleTimeoutSec $BCDRRemoteSessionIdleTimeoutInSec Trace-Execution "Session disconnected." } else { Trace-Execution "Starting a new disconnected PSSession" # Set the idle timeout to 4hrs, which is the same timeout as the entire backup action plan, so that the # execution doesn't get killed too soon and the session dies after the timeout expires. $BCDRRemoteSessionIdleTimeoutInMs = $BCDRRemoteSessionIdleTimeoutInSec * 1000 $option = New-PSSessionOption -OutputBufferingMode Drop -IdleTimeout $BCDRRemoteSessionIdleTimeoutInMs $invokeParam = @{} $invokeParam.ComputerName = $ComputerName $invokeParam.Credential = $Credential $invokeParam.InDisconnectedSession = $true $invokeParam.SessionName = $sessionName $invokeParam.SessionOption = $option if (![string]::IsNullOrEmpty($ConfigurationName)) { $invokeParam.ConfigurationName = $ConfigurationName } else { $invokeParam.Authentication = "Credssp" } $invokeParam.ArgumentList = $BCDRRemoteExecutionScriptBlockParams Invoke-Command @invokeParam -ScriptBlock $BCDRRemoteExecutionScriptBlock } # Try to get session PID and creation time from the latest log $wsPid = $null for ($i = 0; $i -lt 3; $i++) { Trace-Execution "Wait for 10 seconds..." Start-Sleep -s 10 $latestStartLog = Get-BCDRActionEventLog -ComputerName $ComputerName -Credential $Credential ` -BackupID $BackupID -SessionName $sessionName | ? EventID -eq $BCDRRemoteActionStartedEventId | select -First 1 Trace-Execution "Latest start event: $($latestStartLog.Message)" if ($latestStartLog -and ($latestStartLog.Message -Match $sessionInfoEventRegex)) { if ($ProvisionedPSSession -and ($Matches[3] -ne $ProvisionedPSSession.InstanceId)) { Trace-Execution "Waiting for the start action plan event." continue } $wsPid = $Matches[4] $logTime = $latestStartLog.TimeGenerated.ToUniversalTime() break } } if ($i -eq 3) { Trace-Execution "Failed to get PSSession info. Wait for 5 min and try to determine if backup is completed." $timeout = (Get-Date).AddMinutes(5) } while ($true) { # Check for completion $log = Get-BCDRActionEventLog -ComputerName $ComputerName -Credential $Credential ` -BackupID $BackupID -SessionName $sessionName | select -First 1 if ($log) { if ($log.EventId -eq $BCDRRemoteActionCompletededEventId) { Trace-Execution "$ActionType succeeded." break } elseif ($log.EventId -eq $BCDRRemoteActionFailedEventId) { throw "$ActionType $BackupId failed. Message: $($log.Message)" } } # Make sure the process is still alive if ($wsPid) { $sessionProcExists = Check-BCDRSessionProcess -ComputerName $ComputerName -Credential $Credential ` -SessionPid $wsPid -SessionCreationTime $logTime if (!$sessionProcExists) { throw "The remote PSSession unexpectedly died. $ActionType failed." } } if ($timeout -and ((Get-Date) -gt $timeout)) { throw "Could not determine if the $ActionType is finished or not. Failing the $ActionType." } Start-Sleep -Seconds 30 } } finally { if ($ProvisionedPSSessionCredential) { $cleanupCredential = $ProvisionedPSSessionCredential } else { $cleanupCredential = $Credential } if (![string]::IsNullOrEmpty($sessionName)) { $session = Get-PSSession -ComputerName $ComputerName -Credential $cleanupCredential ` -Name $sessionName -ErrorAction SilentlyContinue $session | Remove-PSSession -ErrorAction SilentlyContinue } try { $instance = Get-WSManInstance -ConnectionURI "http://$ComputerName`:5985/wsman" shell -Enumerate ` -cred $cleanupCredential -ErrorAction SilentlyContinue | ? Name -eq $SessionName foreach ($instance in $instances) { Trace-Execution "Removing WSMan objects with shell ID $($instance.ShellId)" Remove-WSManInstance -ConnectionURI "http://$ComputerName`:5985/wsman" shell @{ShellID="$($instance.ShellId)"} ` -cred $cleanupCredential -ErrorAction SilentlyContinue } } catch { Trace-Execution "Could not clean up the WSMan instance. Exception: $_" } } } <# .Synopsis Get the first machine in the list where a remote PSSession can be created. #> function Get-FirstAvailableMachine { param ( [Parameter(Mandatory=$true)] [string[]] $Machines, [Parameter(Mandatory=$true)] [PSCredential] $Credential, [Parameter(Mandatory = $true, ParameterSetName = "JEA")] [ValidateNotNullOrEmpty()] [string] $ConfigurationName, [Parameter(Mandatory = $true, ParameterSetName = "Authentication")] [ValidateSet("Default","Basic","Credssp","Digest","Kerberos","Negotiate","NegotiateWithImplicitCredential")] [string] $Authentication = "Credssp", [Parameter(Mandatory = $false)] [uint32] $Retries = 0, [Parameter(Mandatory = $false)] [uint32] $RetrySleepTimeInSeconds = 5 ) $RemoteSession = $null $found = $null foreach ($Machine in $Machines) { Trace-Execution "Testing remote PSSession connectivities to $Machine with user $($Credential.UserName)" $attempt = 0; try { while ($true) { $attempt++ try { if ($PSCmdlet.ParameterSetName -eq "JEA") { $session = New-PSSession -ComputerName $Machine -Credential $Credential ` -ConfigurationName $ConfigurationName -ErrorAction Stop } else { $session = New-PSSession -ComputerName $Machine -Credential $Credential ` -Authentication $Authentication -ErrorAction Stop } break } catch { Trace-Warning "Test connection failed. Exception: $_" if ($attempt -lt $Retries) { Trace-Execution "Attempt $attempt of $Retries." Start-Sleep -s $RetrySleepTimeInSeconds continue } else { throw "Failed to connect to $Machine after $attempt attempts. Exception: $_" } } } $found = $Machine break } catch { Trace-Warning "Failed to connect to $Machine with exception $($_ | Out-String)" } finally { $session | Remove-PSSession -ErrorAction SilentlyContinue } } if ([string]::IsNullOrEmpty($found)) { throw "Could not create a PSSession to any of the specified machines." } return $found } Export-ModuleMember -Function Ensure-BCDREventLog Export-ModuleMember -Function Get-BCDRActionPlanConst Export-ModuleMember -Function Get-BCDRActionPlanConsts Export-ModuleMember -Function Get-FirstAvailableMachine Export-ModuleMember -Function Start-BCDRExecutionOnRemoteComputer Export-ModuleMember -Function Start-IdempotentBCDRRemoteExecution # SIG # Begin signature block # MIInvwYJKoZIhvcNAQcCoIInsDCCJ6wCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDUkI9CuCg+ewEj # UjnWERSejhLSTok6F5j/AfKseK3mWKCCDXYwggX0MIID3KADAgECAhMzAAADrzBA # DkyjTQVBAAAAAAOvMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMxMTE2MTkwOTAwWhcNMjQxMTE0MTkwOTAwWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDOS8s1ra6f0YGtg0OhEaQa/t3Q+q1MEHhWJhqQVuO5amYXQpy8MDPNoJYk+FWA # hePP5LxwcSge5aen+f5Q6WNPd6EDxGzotvVpNi5ve0H97S3F7C/axDfKxyNh21MG # 0W8Sb0vxi/vorcLHOL9i+t2D6yvvDzLlEefUCbQV/zGCBjXGlYJcUj6RAzXyeNAN # xSpKXAGd7Fh+ocGHPPphcD9LQTOJgG7Y7aYztHqBLJiQQ4eAgZNU4ac6+8LnEGAL # go1ydC5BJEuJQjYKbNTy959HrKSu7LO3Ws0w8jw6pYdC1IMpdTkk2puTgY2PDNzB # tLM4evG7FYer3WX+8t1UMYNTAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQURxxxNPIEPGSO8kqz+bgCAQWGXsEw # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMTgyNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAISxFt/zR2frTFPB45Yd # mhZpB2nNJoOoi+qlgcTlnO4QwlYN1w/vYwbDy/oFJolD5r6FMJd0RGcgEM8q9TgQ # 2OC7gQEmhweVJ7yuKJlQBH7P7Pg5RiqgV3cSonJ+OM4kFHbP3gPLiyzssSQdRuPY # 1mIWoGg9i7Y4ZC8ST7WhpSyc0pns2XsUe1XsIjaUcGu7zd7gg97eCUiLRdVklPmp # XobH9CEAWakRUGNICYN2AgjhRTC4j3KJfqMkU04R6Toyh4/Toswm1uoDcGr5laYn # TfcX3u5WnJqJLhuPe8Uj9kGAOcyo0O1mNwDa+LhFEzB6CB32+wfJMumfr6degvLT # e8x55urQLeTjimBQgS49BSUkhFN7ois3cZyNpnrMca5AZaC7pLI72vuqSsSlLalG # OcZmPHZGYJqZ0BacN274OZ80Q8B11iNokns9Od348bMb5Z4fihxaBWebl8kWEi2O # PvQImOAeq3nt7UWJBzJYLAGEpfasaA3ZQgIcEXdD+uwo6ymMzDY6UamFOfYqYWXk # ntxDGu7ngD2ugKUuccYKJJRiiz+LAUcj90BVcSHRLQop9N8zoALr/1sJuwPrVAtx # HNEgSW+AKBqIxYWM4Ev32l6agSUAezLMbq5f3d8x9qzT031jMDT+sUAoCw0M5wVt # CUQcqINPuYjbS1WgJyZIiEkBMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGZ8wghmbAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAAOvMEAOTKNNBUEAAAAAA68wDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIA/pehCkQwZn7fSFbhJC5ihv # 1vSG/qMNCVDv0o8RlW2FMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAP5WXIramWKXzt1rxRAbufjbgfsY0l2P5dLQ6SSkMt+BJw2LMUs+tNZFU # Na7nXSfYq0silQiu7p7FOStujjAD2+Z0dQJgeG6LCc1pFDlVET/sOR0+fcmm1jPR # SIVy9BL61VWUjvixxzBFjbmQE66Eb15/RwkDUGtZQtLcPiEFCu1qPp+GUX/+9NGD # JX8TMHiBYQPltmuTfmws4BkkPBxFI1QQk8T7e4qxK2nZLeBBGdZWRpbZ8T3mpE3F # 43ZwkCiqut8PyxiFPkzUhknV0xx9UQyLeUBSsocQW/XeFXkERkkZjQYbNbkNvPvF # TBgzvxzB6PynFQlStJBCHPO4zR3Ob6GCFykwghclBgorBgEEAYI3AwMBMYIXFTCC # FxEGCSqGSIb3DQEHAqCCFwIwghb+AgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFZBgsq # hkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCBuqLzBhl3nsSGtLKOIH966A9AMcs58S+iKMiXp9rHmlgIGZbqk0R90 # GBMyMDI0MDIxMjE0MDgzNy4wMzdaMASAAgH0oIHYpIHVMIHSMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl # bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNO # OkZDNDEtNEJENC1EMjIwMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT # ZXJ2aWNloIIReDCCBycwggUPoAMCAQICEzMAAAHimZmV8dzjIOsAAQAAAeIwDQYJ # KoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcNMjMx # MDEyMTkwNzI1WhcNMjUwMTEwMTkwNzI1WjCB0jELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxhbmQgT3Bl # cmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjpGQzQxLTRC # RDQtRDIyMDElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC # AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALVjtZhV+kFmb8cKQpg2mzis # DlRI978Gb2amGvbAmCd04JVGeTe/QGzM8KbQrMDol7DC7jS03JkcrPsWi9WpVwsI # ckRQ8AkX1idBG9HhyCspAavfuvz55khl7brPQx7H99UJbsE3wMmpmJasPWpgF05z # ZlvpWQDULDcIYyl5lXI4HVZ5N6MSxWO8zwWr4r9xkMmUXs7ICxDJr5a39SSePAJR # IyznaIc0WzZ6MFcTRzLLNyPBE4KrVv1LFd96FNxAzwnetSePg88EmRezr2T3HTFE # lneJXyQYd6YQ7eCIc7yllWoY03CEg9ghorp9qUKcBUfFcS4XElf3GSERnlzJsK7s # /ZGPU4daHT2jWGoYha2QCOmkgjOmBFCqQFFwFmsPrZj4eQszYxq4c4HqPnUu4hT4 # aqpvUZ3qIOXbdyU42pNL93cn0rPTTleOUsOQbgvlRdthFCBepxfb6nbsp3fcZaPB # fTbtXVa8nLQuMCBqyfsebuqnbwj+lHQfqKpivpyd7KCWACoj78XUwYqy1HyYnStT # me4T9vK6u2O/KThfROeJHiSg44ymFj+34IcFEhPogaKvNNsTVm4QbqphCyknrwBy # qorBCLH6bllRtJMJwmu7GRdTQsIx2HMKqphEtpSm1z3ufASdPrgPhsQIRFkHZGui # hL1Jjj4Lu3CbAmha0lOrAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQURIQOEdq+7Qds # lptJiCRNpXgJ2gUwHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD # VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j # cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG # CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw # MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD # CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAORURDGrVRTbnulf # sg2cTsyyh7YXvhVU7NZMkITAQYsFEPVgvSviCylr5ap3ka76Yz0t/6lxuczI6w7t # Xq8n4WxUUgcj5wAhnNorhnD8ljYqbck37fggYK3+wEwLhP1PGC5tvXK0xYomU1nU # +lXOy9ZRnShI/HZdFrw2srgtsbWow9OMuADS5lg7okrXa2daCOGnxuaD1IO+65E7 # qv2O0W0sGj7AWdOjNdpexPrspL2KEcOMeJVmkk/O0ganhFzzHAnWjtNWneU11WQ6 # Bxv8OpN1fY9wzQoiycgvOOJM93od55EGeXxfF8bofLVlUE3zIikoSed+8s61NDP+ # x9RMya2mwK/Ys1xdvDlZTHndIKssfmu3vu/a+BFf2uIoycVTvBQpv/drRJD68eo4 # 01mkCRFkmy/+BmQlRrx2rapqAu5k0Nev+iUdBUKmX/iOaKZ75vuQg7hCiBA5xIm5 # ZIXDSlX47wwFar3/BgTwntMq9ra6QRAeS/o/uYWkmvqvE8Aq38QmKgTiBnWSS/uV # PcaHEyArnyFh5G+qeCGmL44MfEnFEhxc3saPmXhe6MhSgCIGJUZDA7336nQD8fn4 # y6534Lel+LuT5F5bFt0mLwd+H5GxGzObZmm/c3pEWtHv1ug7dS/Dfrcd1sn2E4gk # 4W1L1jdRBbK9xwkMmwY+CHZeMSvBMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ # mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT # Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m # dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh # dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1 # WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB # BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK # NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg # fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp # rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d # vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9 # 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR # Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu # qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO # ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb # oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6 # bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t # AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW # BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb # UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku # aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA # QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2 # VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu # bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw # LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93 # d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt # MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q # XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6 # U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt # I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis # 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp # kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0 # sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e # W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ # sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7 # Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0 # dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ # tB1VM1izoXBm8qGCAtQwggI9AgEBMIIBAKGB2KSB1TCB0jELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxh # bmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjpG # QzQxLTRCRDQtRDIyMDElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy # dmljZaIjCgEBMAcGBSsOAwIaAxUAFpuZafp0bnpJdIhfiB1d8pTohm+ggYMwgYCk # fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF # AOl0S1kwIhgPMjAyNDAyMTIxNTQ2MzNaGA8yMDI0MDIxMzE1NDYzM1owdDA6Bgor # BgEEAYRZCgQBMSwwKjAKAgUA6XRLWQIBADAHAgEAAgIVwzAHAgEAAgIUTzAKAgUA # 6XWc2QIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIBAAID # B6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBAKSvdq5/OwSqLipXDPR/ # fz5CiLbm0lCxv+PjDUqfK5ccPWEVkKsD9zmiB4N1ZQarjww5WUro+QdoUzSzV/Ly # OaxhyB8c2JwDLLMQK8JyHWefD6cG4cvgYDg5iEzioerBrjSnrGxsb3qVV0n31Fhc # Z9Hh71O9RocFcjpVHRSVEF7MMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUt # U3RhbXAgUENBIDIwMTACEzMAAAHimZmV8dzjIOsAAQAAAeIwDQYJYIZIAWUDBAIB # BQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0BCQQx # IgQgG/gaQA83XbdjJX6LwoAs4jDPRvJCpQ8MFXb0IV8tgdkwgfoGCyqGSIb3DQEJ # EAIvMYHqMIHnMIHkMIG9BCAriSpKEP0muMbBUETODoL4d5LU6I/bjucIZkOJCI9/ # /zCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw # DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x # JjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB4pmZ # lfHc4yDrAAEAAAHiMCIEII1oQoIqDDF/dls2LazpjkRTP0GoZvbNELQJtb8XKdao # MA0GCSqGSIb3DQEBCwUABIICAGx0QXlFPYRkc38TSskobXPwiWkrN3L96VNsbE/q # Q9XabTqJpeUg4lvE4xcvX05Nqdks7F0e2zqiWugAfTU6lRj+D+EmxFugKzmABK4/ # 0ylPLARi+rKwz8QNWvAKhzdL6XcGYK0PtPcRjoUrG/l/H5kZt9KVYWopQpsE+jlR # 6cl+z91e2O0UnwXnpggiYS/LLtEF4uFzScq2vzwO6/Y0qcJhGOQWodnnWC21vSZc # fWH50wTNp1sUYTGL4X4vRl5b49rxRSY1bv0IB2mTGfLedafBB/sADSf4BvonAaZ+ # WC6S6KTpKX2o8KYojAlXXAjDGVwB+J8TLoWxyQ/i+MtPhqSCL9GluR0Vr4C3fGjd # ntxua+sRVulupYycVr1USnB0RSsextvahtwxvay14aJP4NSsk7Xu7oy1KjfODmZm # w8ZLf6rY6+coT14aSuNNQMXdmPMJDl7oPAArD1XMtqu4u5jmOgI0ffGweQj4YSIt # 5nsPh8BlIOZcz2oke6r8j01pCPaYB9qp6a0En+FCK8U/cR4nF3qwsfeJwQnbUV/Y # skHwjl5rR+NV7IShm1NL7bJoRdR0TI8WojLHtK9ZIwcXASUgat5lCSr1FCKKDsy1 # DcKDuSqsG0RCABnbd6h8DIHNfgSDDptPQIDtuWKwni61vDfiuyOTNOtvXivqHHG9 # vbWP # SIG # End signature block |