Obs/bin/ObsDep/content/Powershell/Common/BCDRActionHelper.psm1

<###################################################
 # #
 # Copyright (c) Microsoft. All rights reserved. #
 # #
 ##################################################>


 Import-Module $PSScriptRoot\Tracer.psm1

 $BackupStartedEventLogPattern = "Action {0} started with session name <{1}>, session shell ID {2} and process ID {3}."
 $BackupCompletedEventLogPattern = "Action {0} completed with session name <{1}>."
 $BackupFailedEventLogPattern = "Action {0} failed with session name <{1}>. Exception: {2}"
 $BCDRSessionNameBase = "AzsBCDRSession"
 $BCDRLogName = "AzSBCDR"
 $BCDRBackupEventSource = "AzSBackupAction"

 # TODO: restore actions are not utilizing the action helper at the moment.
 $BCDRRestoreEventSource = "AzSRestoreAction"

 $BCDRRemoteActionStartedEventId = 1
 $BCDRRemoteActionCompletededEventId = 2
 $BCDRRemoteActionFailedEventId = 3

 # 4 hour session idle timeout
 $BCDRRemoteSessionIdleTimeoutInSec = 14400

<#
.Synopsis
   Get the BCDR action plan constants
#>

function Get-BCDRActionPlanConsts
{
    $consts = @{}
    $consts.BackupStartedEventLogPattern = $BackupStartedEventLogPattern
    $consts.BackupCompletedEventLogPattern = $BackupCompletedEventLogPattern
    $consts.BackupFailedEventLogPattern = $BackupFailedEventLogPattern
    $consts.BCDRSessionNameBase = $BCDRSessionNameBase
    $consts.BCDRLogName = $BCDRLogName
    $consts.BCDRBackupEventSource = $BCDRBackupEventSource
    $consts.BCDRRestoreEventSource = $BCDRRestoreEventSource
    $consts.BCDRRemoteActionStartedEventId = $BCDRRemoteActionStartedEventId
    $consts.BCDRRemoteActionCompletededEventId = $BCDRRemoteActionCompletededEventId
    $consts.BCDRRemoteActionFailedEventId = $BCDRRemoteActionFailedEventId

    return $consts
}

<#
.Synopsis
   Get the BCDR action plan constant
#>

function Get-BCDRActionPlanConst
{
    [CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [ValidateSet('BackupStartedEventLogPattern', 'BackupCompletedEventLogPattern', 'BackupFailedEventLogPattern', `
            'BCDRSessionNameBase', 'BCDRLogName', 'BCDRBackupEventSource', 'BCDRRestoreEventSource', `
            'BCDRRemoteActionStartedEventId', 'BCDRRemoteActionCompletededEventId', 'BCDRRemoteActionFailedEventId')]
        [string] $ConstName
    )

    $consts = Get-BCDRActionPlanConsts
    return $consts[$ConstName]
}

<#
.Synopsis
    Create event source and log on target VMs
#>

function Ensure-BCDREventLog
{
    param
    (
        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $ComputerName,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [PSCredential] $Credential,

        [Parameter(Mandatory=$true)]
        [string] $EventSource
    )

    Invoke-Command -ComputerName $ComputerName -Credential $Credential -ArgumentList @($BCDRLogName, $EventSource) `
        -ScriptBlock {
        param ($BCDRLogName, $EventSource)
        if ([System.Diagnostics.EventLog]::SourceExists($EventSource) -eq $false)
        {
            Trace-Execution "Creating event source $EventSource on event log $BCDRLogName"
            [System.Diagnostics.EventLog]::CreateEventSource($EventSource, $BCDRLogName)
            Trace-Execution "Event source $EventSource created"
            Limit-EventLog -LogName $BCDRLogName -RetentionDays 30
        }
    }
}

<#
.Synopsis
    Get BCDR action event logs with a specific backup from the target VM
#>

function Get-BCDRActionEventLog
{
    param 
    (
        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $ComputerName,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [PSCredential] $Credential,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $BackupId,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $SessionName
    )

    return Invoke-Command -ComputerName $ComputerName -Credential $Credential -ArgumentList @($BackupId, $BCDRLogName, $SessionName) -ScriptBlock {
        param ($BackupId, $BCDRLogName, $SessionName)
        try
        {
            $logs = Get-EventLog -LogName $BCDRLogName -ErrorAction Stop `
                | ? { ($_.Message -like "*$BackupId*") -and ($_.Message -like "*<$SessionName>*") }

            # Log the last 5 logs about this backup session for diagnostics
            $numHistory = 5
            foreach ($log in ($logs | select -First $numHistory))
            {
                Trace-Execution "Last $numHistory logs for backup $BackupId and session '$SessionName'"
                Trace-Execution "$($log.TimeGenerated), EventId: $($log.EventID), Msg: $($log.Message)"
            }

            return $logs
        }
        catch
        {}
    }
}

<#
.Synopsis
    Check if the BCDR remote session process still exists and kill the process if requested to
#>

function Check-BCDRSessionProcess
{
    param
    (
        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $ComputerName,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [PSCredential] $Credential,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $SessionPid,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [DateTime] $SessionCreationTime,

        [switch] $Kill
    )

    $proc = $null
    $retries = 10
    while (!$proc)
    {
        try
        {
            # Use Invoke-Command instead of 'Get-Process -CompueterName' to make sure process StartTime properties can
            # be returned
            $proc = Invoke-Command -ComputerName $ComputerName -Credential $Credential -ScriptBlock {
                Get-Process -ProcessName 'wsmprovhost' -ErrorAction Stop
            }
        }
        catch
        {
            $retries--
            if ($retries -gt 0)
            {
                Trace-Execution "Failed to get PowerShell remote host processes on $ComputerName. Exception: $_. Retrying..."
                Start-Sleep -s 10
                continue
            }

            throw
        }
    }

    $sessionProc = $proc | ? { ($_.Id -eq $SessionPid)}
    if ($sessionProc)
    {
        # Check the creation time to prevent accidental termination of other processes with the same Pid.
        # Session process must be started earlier than session log time. Give it 1 extra second to the session creation
        # time because ETL log time has seconds precision while the process start time has milliseconds precision.
        $sessionProcStartTimeUTC = $sessionProc.StartTime.ToUniversalTime()
        $sessionCreationTimeUTC = $SessionCreationTime.ToUniversalTime().AddSeconds(1)

        if ($sessionProcStartTimeUTC -le $sessionCreationTimeUTC)
        {
            if ($Kill.IsPresent)
            {
                Invoke-Command -ComputerName $ComputerName -Credential $Credential -ArgumentList @($SessionPid) `
                    -ScriptBlock {
                        param ($SessionPid)
                        Stop-Process -Id $SessionPid -Force -ErrorAction SilentlyContinue
                }
            }

            return $true
        }
        else
        {
            Trace-Execution "The process with Id $SessionPid isn't the remote PowerShell host process for this invocation."
            Trace-Execution "Session process start time: $sessionProcStartTimeUTC, Session creation time: $sessionCreationTimeUTC"
        }
    }
    else
    {
        $str = $null
        $proc | select Id, ProcessName | % { $str += "($($_.Id), $($_.ProcessName))`n" }
        Trace-Execution "All PowerShell remote host processes names on $ComputerName :`n$str"
        Trace-Execution "Failed to find a remote PowerShell host process with process Id $SessionPid on $ComputerName."
    }

    return $false
}

<#
.Synopsis
    Check previously started BCDR remote sessions.
 
    The BCDR remote sessions for each role (and each invocation if a role has more than one) are named with this pattern:
    '<BCDRSessionNameBase>-<FullRepositoryName>'
 
    This method searches the remote session name on all nodes for the current role, try to get the action status to
    determine next actions, and finally clean up the sessions.
#>

function Check-PreviousBCDRSession
{
    param
    (
        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string[]] $NodesForCurrentRole,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [PSCredential] $Credential,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $BackupId,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $SessionName,

        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $SessionInfoEventRegex,

        [Parameter(Mandatory=$false)]
        $ProvisionedPSSession = $null,

        [Parameter(Mandatory=$false)]
        [PSCredential] $ProvisionedPSSessionCredential = $null,

        [Parameter(Mandatory=$true)]
        [uint32] $ExpectedRuntimeInMin,

        [Parameter(Mandatory=$true)]
        [ref] $ShouldInvokeAction
    )

    $ShouldInvokeAction.Value = $true

    # Test if the session already exists on all the nodes
    $sessions = @()
    Trace-Execution "Searching for existing PSSessions on $NodesForCurrentRole"
    foreach ($Node in $NodesForCurrentRole)
    {
        if ($ProvisionedPSSessionCredential)
        {
            $sessions += Get-PSSession -ComputerName $Node -Credential $ProvisionedPSSessionCredential -Name $SessionName -ErrorAction SilentlyContinue
        }
        else
        {
            $sessions += Get-PSSession -ComputerName $Node -Credential $Credential -Name $SessionName -ErrorAction SilentlyContinue
        }
    }

    # Make sure to filter out the provisioned PSSession if specified.
    Trace-Execution "Found sessions: $sessions"
    if ($ProvisionedPSSession)
    {
        $sessions = $sessions | ? {
            if ($ProvisionedPSSession.InstanceId -ne $_.InstanceId)
            {
                return $true
            }
            else
            {
                Trace-Execution "Filter out the provisioned PSSession with ID $($ProvisionedPSSession.InstanceId) from the sessions."
                return $false
            }
        }
    }

    foreach ($session in $sessions)
    {
        # Properly working BCDR remote sessions should be in the 'Disconnected' state and ready to be connected
        try
        {
            Trace-Execution "The disconnected remote PSSession '$($session.Name)' exists on $($session.ComputerName)."
            Trace-Execution "State: $($session.State), Availability: $($session.Availability), Instance ID: $($session.InstanceId)."
            Trace-Execution "Check the previous backup status of BackupID $backupID."

            # Get the latest log with this backup ID
            $latestLog = Get-BCDRActionEventLog -ComputerName $session.ComputerName -Credential $Credential `
                -BackupId $backupId -SessionName $SessionName | select -First 1

            if ($latestLog)
            {
                switch ($latestLog.EventId)
                {
                    # Started
                    $BCDRRemoteActionStartedEventId
                    {
                        $expectedCompletion = $latestLog.TimeGenerated.ToUniversalTime().AddMinutes($ExpectedRuntimeInMin)
                        Trace-Execution "Action started at UTC $($latestLog.TimeGenerated.ToUniversalTime().ToString())"
                        Trace-Execution "Expected to complete by UTC $($expectedCompletion.ToUniversalTime().ToString())"
                        $actionPlanDone = $false

                        do
                        {
                            # Check for completion
                            $now = Get-Date
                            Trace-Execution "Current time is UTC $($now.ToUniversalTime().ToString())"

                            $log = Get-BCDRActionEventLog -ComputerName $session.ComputerName -Credential $Credential `
                                -BackupId $backupId -SessionName $SessionName | select -First 1
                            if ($log.EventId -eq $BCDRRemoteActionCompletededEventId)
                            {
                                $ShouldInvokeAction.Value = $false
                                $actionPlanDone = $true
                            }
                            elseif ($log.EventId -eq $BCDRRemoteActionFailedEventId)
                            {
                                Trace-Execution "The previous action failed, invoking the action again"
                                $actionPlanDone = $true
                            }

                            if ($actionPlanDone)
                            {
                                break
                            }

                            # Keep waiting
                            Start-Sleep -Seconds 30

                        } while ($now -le $expectedCompletion)

                        if (!$actionPlanDone)
                        {
                            Trace-Execution "The previous action plan didn't finish in time, restarting another one."
                        }
                    }

                    # Completed
                    $BCDRRemoteActionCompletededEventId { $shouldInvokeAction.Value = $false }
        
                    # Failed
                    $BCDRRemoteActionFailedEventId { Trace-Execution "The previous action failed, invoking the action again" }
                }
            }

            break
        }
        catch
        {}
    }

    if ($sessions)
    {
        Trace-Execution "Removing the disconnected PSSessions."
        $sessions | Remove-PSSession -ErrorAction SilentlyContinue
    }

    foreach ($Node in $NodesForCurrentRole)
    {
        try
        {
            Trace-Execution "Make sure the WSMan instance and the process are both gone on $Node"
            $instances = Get-WSManInstance -ConnectionURI "http://$Node`:5985/wsman" shell -Enumerate `
                -cred $Credential -ErrorAction SilentlyContinue | ? Name -eq $SessionName
    
            if ($ProvisionedPSSession)
            {
                $instances = $instances | ? {
                    if ($ProvisionedPSSession.InstanceId -ne $_.ShellId)
                    {
                        return $true
                    }
                    else
                    {
                        Trace-Execution "Filter out the provisioned PSSession ID $($ProvisionedPSSession.InstanceId) from the instances"
                        return $false
                    }
                }
            }
    
            foreach ($instance in $instances)
            {
                Trace-Execution "Try to remove the WSMan object with shell ID $($instance.ShellId)"
                Remove-WSManInstance -ConnectionURI "http://$Node`:5985/wsman" shell @{ShellID="$($instance.ShellId)"} `
                    -cred $Credential -ErrorAction SilentlyContinue
            }

            $logs = Get-BCDRActionEventLog -ComputerName $Node -Credential $Credential -BackupId $backupId `
                -SessionName $SessionName | ? EventID -eq $BCDRRemoteActionStartedEventId

            if ($logs)
            {
                Trace-Execution "Check the previous session PID."
                foreach ($log in $logs)
                {
                    Trace-Execution "Log message: $($log.Message)"
                    if ($log.Message -Match $SessionInfoEventRegex)
                    {
                        $sessionNameFromLog = $Matches[2]
                        if ($sessionNameFromLog -ne $SessionName)
                        {
                            Trace-Execution "Not for the current session, skipped."
                            continue
                        }
    
                        $wsPid = $Matches[4]
                        $logTime = $log.TimeGenerated.ToUniversalTime()
                        if (![string]::IsNullOrEmpty($wsPid))
                        {
                            Trace-Execution "Make sure process '$wsPid' is gone."
                            $null = Check-BCDRSessionProcess -ComputerName $Node -Credential $Credential `
                                -SessionPid $wsPid -SessionCreationTime $logTime -Kill
                        }
                    }
                }
            }
        }
        catch
        {
            Trace-Execution "Failed to clean up the WSMan instance and/or the process on $Node. Proceeding anyway. Exception: $_"
        }
    }
}

<#
.Synopsis
    A wrapper around actual execution on remote machines when the execution runs in a JEA session, in which case this
    method must be whitelisted in the session configuration (.psrc) or imported manually as part of the initialization
#>

function Start-BCDRExecutionOnRemoteComputer
{
    param
    (
        [Parameter(Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $BCDRRemoteExecutionScriptBlockAsStr,

        [Parameter(Mandatory=$true)]
        $ScriptBlockParameters
    )

    $BCDRRemoteExecutionScriptBlock = [scriptblock]::Create($BCDRRemoteExecutionScriptBlockAsStr)

    # Start the script block execution async
    Start-Job -ScriptBlock $BCDRRemoteExecutionScriptBlock -ArgumentList $ScriptBlockParameters
}

<#
.Synopsis
    Execute backup operations idempotently.
 
    BCDR ECE actions start remote PSSession on VMs for roles that implements backup/restore interfaces to perform
    backup/restore actions. This method ensures that these sessions can be tracked or cleanly canceled when ECE fails
    over and/or crashes by disconnecting the worker PSSessions and poll for completion.
 
.PARAMETER ComputerName
    The targe computer name where the execution is run. If ProvisionedPSSession isn't specified, a disconnected session
    will be created for the caller. Otherwise the computer name is used to poll action plan results.
 
.PARAMETER Credential
    The credential to access the computer.
 
.PARAMETER BackupID
    Backup ID
 
.PARAMETER FullRepositoryName
    The full repository name is used to construct the PSSession name, which should be distinct for each repository so
    that they can be tracked individually. The pattern is 'AzsBCDRSession-<FullRepostiroyName>'
 
.PARAMETER NodesForCurrentRole
    List of Nodes for the role where the method should look for previously started BCDR remote PSSessions
 
.PARAMETER EmbeddedScriptBlockAsStr
    The script block that contains the codes for the actual execution on the remote machine, passed in as a string.
    The author of the script block must ensure that the codes runs in constrained language mode, otherwise the author
    must manually set full language mode in this script block.
 
.PARAMETER ScriptBlockParameters
    The parameter list of the embedded script block
 
.PARAMETER ProvisionedPSSession
    The provisioned PSSession. The session name must follow this pattern: 'AzsBCDRSession-<FullRepositoryName>'.
    Once the codes in the EmbeddedScriptBlockAsStr is executed as a job in the session, the session will be disconnected
    and the caller should *NOT* try to reconnect to it if a reference is still held.
 
.PARAMETER ExpectedRuntimeInMin
    The method first checks if there were any previously started PSSessions, and if there is any, it finds the start
    time of the execution and poll for results until the run exceeds the expected run time.
 
.PARAMETER ConfigurationName
    The configuration name to start the disconnected session with. If not specified, default to use CredSSP
 
.PARAMETER ActionType
    Specifies whether the action is a backup or restore
#>

function Start-IdempotentBCDRRemoteExecution
{
    param
    (
        [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)]
        [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $ComputerName,

        [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)]
        [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [PSCredential] $Credential,

        [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)]
        [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $BackupID,

        [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)]
        [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$false)]
        [ValidateNotNullOrEmpty()]
        [string] $FullRepositoryName,

        [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)]
        [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string[]] $NodesForCurrentRole,

        [Parameter(ParameterSetName = "DefaultSet", Mandatory=$true)]
        [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        [string] $EmbeddedScriptBlockAsStr,

        [Parameter(Mandatory=$false)]
        [Array] $ScriptBlockParameters,

        [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)]
        [ValidateNotNullOrEmpty()]
        $ProvisionedPSSession = $null,

        [Parameter(ParameterSetName = "ProvisionedPSSessionSet", Mandatory=$true)]
        [ValidateNotNull()]
        [PSCredential] $ProvisionedPSSessionCredential = $null,

        [Parameter(Mandatory=$false)]
        [uint32] $ExpectedRuntimeInMin = 60 * 4,

        [Parameter(ParameterSetName = "DefaultSet", Mandatory=$false)]
        [string] $ConfigurationName = $null,

        [Parameter(Mandatory = $false)]
        [ValidateSet('Backup', 'Restore')]
        [string] $ActionType = 'Backup',

        [Parameter(Mandatory = $false)]
        [string] $BackupSnapshotFolder,

        [Parameter(Mandatory = $false)]
        [string[]] $BackupSnapshotFilters
    )

    try
    {
        $defaultSessionName = "$BCDRSessionNameBase-$FullRepositoryName"
        if ($ProvisionedPSSession)
        {
            if ($ProvisionedPSSession.Name -notlike "$defaultSessionName*")
            {
                throw "Unexpected PSSession name '$($ProvisionedPSSession.Name)'. The session must start with '$defaultSessionName'."
            }

            $sessionName = $ProvisionedPSSession.Name
        }
        else
        {
            $sessionName = $defaultSessionName
        }

        if ($ActionType -eq 'Backup')
        {
            $EventSource = $BCDRBackupEventSource
        }
        else
        {
            $EventSource = $BCDRRestoreEventSource
        }

        $sessionInfoEventRegex = $BackupStartedEventLogPattern -f "([\d\w\-]+)", "(.+)", "([\d\w\-]+)", "(\d+)"

        $shouldInvokeAction = $true
        Check-PreviousBCDRSession -NodesForCurrentRole $NodesForCurrentRole -Credential $Credential -BackupID $BackupID `
            -SessionName $sessionName -SessionInfoEventRegex $sessionInfoEventRegex `
            -ProvisionedPSSession $ProvisionedPSSession -ProvisionedPSSessionCredential $ProvisionedPSSessionCredential `
            -ExpectedRuntimeInMin $ExpectedRuntimeInMin -ShouldInvokeAction ([ref] $shouldInvokeAction)

        if ($shouldInvokeAction -eq $false)
        {
            Trace-Execution "The previous action is done, skip invoking the action again"
            return
        }

        Trace-Execution "Ensure the BCDR event source and log exists"
        Ensure-BCDREventLog -ComputerName $ComputerName -Credential $Credential -EventSource $EventSource

        $BCDREventParams = @{}
        $BCDREventParams.BCDRLogName = $BCDRLogName
        $BCDREventParams.EventSource = $EventSource
        $BCDREventParams.BCDRRemoteActionStartedEventId = $BCDRRemoteActionStartedEventId
        $BCDREventParams.BCDRRemoteActionCompletededEventId = $BCDRRemoteActionCompletededEventId
        $BCDREventParams.BCDRRemoteActionFailedEventId = $BCDRRemoteActionFailedEventId
        $BCDREventParams.BackupStartedEventLogPattern = $BackupStartedEventLogPattern
        $BCDREventParams.BackupCompletedEventLogPattern = $BackupCompletedEventLogPattern
        $BCDREventParams.BackupFailedEventLogPattern = $BackupFailedEventLogPattern

        $BCDRRemoteExecutionScriptBlockParams = @($BCDREventParams, $sessionName, $BackupID, `
            $EmbeddedScriptBlockAsStr, $ScriptBlockParameters, $BackupSnapshotFolder, $BackupSnapshotFilters)

        $BCDRRemoteExecutionScriptBlock = {
            param ($BCDREventParams, $sessionName, $BackupID, $EmbeddedScriptBlockAsStr, $ScriptBlockParameters, $BackupSnapshotFolder, $BackupSnapshotFilters)

            try
            {
                Import-Module OpenUpSession
                Set-FullLanguage
            }
            catch{}

            # Log the shell ID and the PID of the disconnected session
            $instance = Get-WSManInstance -ConnectionURI "http://localhost:5985/wsman" shell -Enumerate `
                | Where-Object Name -eq $sessionName
            $backupStartedEventLogMsg = $BCDREventParams.BackupStartedEventLogPattern -f $BackupID, $sessionName, `
                $instance.ShellId, $instance.ProcessId
            Write-EventLog -LogName $BCDREventParams.BCDRLogName -Source $BCDREventParams.EventSource `
                -EntryType Information -EventId $BCDREventParams.BCDRRemoteActionStartedEventId -Message $backupStartedEventLogMsg

            $sb = [scriptblock]::Create($EmbeddedScriptBlockAsStr)
            $succeeded = $false
            try
            {
                Invoke-Command -ArgumentList $ScriptBlockParameters -ScriptBlock $sb -ErrorAction Stop
                $succeeded = $true
            }
            catch
            {
                $errorRecord = $_
            }
            finally
            {
                $errorMsg = $errorRecord | select * | Out-String

                # Double check the snapshots to adjust the final result
                if ($BackupSnapshotFolder -and ($BackupSnapshotFilters.Count -gt 0))
                {
                    try
                    {
                        $logFileFullPath = Join-Path  "$env:SystemDrive\MASLogs" -ChildPath "$($sessionName)_$($BackupID)_$(Get-Date -Format "yyyyMMdd-HHmmss").log"
                        Start-Transcript -Append -Path $logFileFullPath

                        Write-Verbose "Check snapshots under $BackupSnapshotFolder with $BackupSnapshotFilters finally to adjust result"
                        $snapshotNotFound = $false
                        foreach ($filter in $BackupSnapshotFilters)
                        {
                            $snapshots = Get-ChildItem $BackupSnapshotFolder -Filter $filter -Force
                            if (!$snapshots)
                            {
                                Write-Verbose "Failed to find snapshot with $filter"
                                $succeeded = $false
                                $snapshotNotFound = $true
                                $errorMsg += "`nNo snapshot found under $BackupSnapshotFolder with filter $filter."
                            }
                        }

                        if (!$snapshotNotFound)
                        {
                            Write-Verbose "Succeeded to find all snapshots, adjust result from $succeeded to true"
                            $succeeded = $true
                        }
                    }
                    catch
                    {
                        Write-Verbose "Ignore exception $($_.Exception.Message) when checking snapshots"
                    }
                    finally
                    {
                        Stop-Transcript -ErrorAction Ignore
                    }
                }
                
                if ($succeeded)
                {
                    $backupCompletedEventLogMsg = $BCDREventParams.BackupCompletedEventLogPattern -f $BackupID, $sessionName
                    Write-EventLog -LogName $BCDREventParams.BCDRLogName -Source $BCDREventParams.EventSource `
                        -EntryType Information -EventId $BCDREventParams.BCDRRemoteActionCompletededEventId `
                        -Message $backupCompletedEventLogMsg
                }
                else
                {
                    $backupFailedEventLogMsg = $BCDREventParams.BackupFailedEventLogPattern -f $BackupID, $sessionName, $errorMsg
                    Write-EventLog -LogName $BCDREventParams.BCDRLogName -Source $BCDREventParams.EventSource `
                        -EntryType Error -EventId $BCDREventParams.BCDRRemoteActionFailedEventId `
                        -Message $backupFailedEventLogMsg
                    throw $errorMsg, $errorRecord
                }
            }
        }

        if ($PsCmdlet.ParameterSetName -eq 'ProvisionedPSSessionSet')
        {
            Trace-Execution "Using the provisioned PSSession."
            Trace-Execution "Name: $($ProvisionedPSSession.Name), State: $($ProvisionedPSSession.State)"
            Trace-Execution "Availability: $($ProvisionedPSSession.Availability), ConfigName: $($ProvisionedPSSession.ConfigurationName)"

            # Execute the script block in the provisioned session and disconnect
            #
            # Most cases where the caller passes in a provisioned PSSession is because the session was created with a
            # JEA endpoint and specific language mode is enforced. Therefore, Start-BCDRExecutionOnRemoteComputer must
            # be imported in the provisioned PSSession already and cannot be imported here to support this scenario.
            Invoke-Command -Session $ProvisionedPSSession `
                -ArgumentList @($BCDRRemoteExecutionScriptBlock.ToString(), $BCDRRemoteExecutionScriptBlockParams) `
                -ScriptBlock {
                    param ($BCDRRemoteExecutionScriptBlockAsStr, $BCDRRemoteExecutionScriptBlockParams)
                    Start-BCDRExecutionOnRemoteComputer `
                        -BCDRRemoteExecutionScriptBlockAsStr $BCDRRemoteExecutionScriptBlockAsStr `
                        -ScriptBlockParameters $BCDRRemoteExecutionScriptBlockParams
                }

            Trace-Execution "Disconnecting the remote PSSession."
            Disconnect-PSSession -Session $ProvisionedPSSession -IdleTimeoutSec $BCDRRemoteSessionIdleTimeoutInSec
            Trace-Execution "Session disconnected."
        }
        else
        {
            Trace-Execution "Starting a new disconnected PSSession"

            # Set the idle timeout to 4hrs, which is the same timeout as the entire backup action plan, so that the
            # execution doesn't get killed too soon and the session dies after the timeout expires.
            $BCDRRemoteSessionIdleTimeoutInMs = $BCDRRemoteSessionIdleTimeoutInSec * 1000
            $option = New-PSSessionOption -OutputBufferingMode Drop -IdleTimeout $BCDRRemoteSessionIdleTimeoutInMs

            $invokeParam = @{}
            $invokeParam.ComputerName = $ComputerName
            $invokeParam.Credential = $Credential
            $invokeParam.InDisconnectedSession = $true
            $invokeParam.SessionName = $sessionName
            $invokeParam.SessionOption = $option

            if (![string]::IsNullOrEmpty($ConfigurationName))
            {
                $invokeParam.ConfigurationName = $ConfigurationName
            }
            else
            {
                $invokeParam.Authentication = "Credssp"
            }

            $invokeParam.ArgumentList = $BCDRRemoteExecutionScriptBlockParams
            Invoke-Command @invokeParam -ScriptBlock $BCDRRemoteExecutionScriptBlock
        }

        # Try to get session PID and creation time from the latest log
        $wsPid = $null
        for ($i = 0; $i -lt 3; $i++)
        {
            Trace-Execution "Wait for 10 seconds..."
            Start-Sleep -s 10

            $latestStartLog = Get-BCDRActionEventLog -ComputerName $ComputerName -Credential $Credential `
                -BackupID $BackupID -SessionName $sessionName | ? EventID -eq $BCDRRemoteActionStartedEventId | select -First 1

            Trace-Execution "Latest start event: $($latestStartLog.Message)"
            if ($latestStartLog -and ($latestStartLog.Message -Match $sessionInfoEventRegex))
            {
                if ($ProvisionedPSSession -and ($Matches[3] -ne $ProvisionedPSSession.InstanceId))
                {
                    Trace-Execution "Waiting for the start action plan event."
                    continue
                }

                $wsPid = $Matches[4]
                $logTime = $latestStartLog.TimeGenerated.ToUniversalTime()
                break
            }
        }

        if ($i -eq 3)
        {
            Trace-Execution "Failed to get PSSession info. Wait for 5 min and try to determine if backup is completed."
            $timeout = (Get-Date).AddMinutes(5)
        }

        while ($true)
        {
            # Check for completion
            $log = Get-BCDRActionEventLog -ComputerName $ComputerName -Credential $Credential `
                -BackupID $BackupID -SessionName $sessionName | select -First 1
            if ($log)
            {
                if ($log.EventId -eq $BCDRRemoteActionCompletededEventId)
                {
                    Trace-Execution "$ActionType succeeded."
                    break
                }
                elseif ($log.EventId -eq $BCDRRemoteActionFailedEventId)
                {
                    throw "$ActionType $BackupId failed. Message: $($log.Message)"
                }
            }

            # Make sure the process is still alive
            if ($wsPid)
            {
                $sessionProcExists = Check-BCDRSessionProcess -ComputerName $ComputerName -Credential $Credential `
                    -SessionPid $wsPid -SessionCreationTime $logTime
                if (!$sessionProcExists)
                {
                    throw "The remote PSSession unexpectedly died. $ActionType failed."
                }
            }

            if ($timeout -and ((Get-Date) -gt $timeout))
            {
                throw "Could not determine if the $ActionType is finished or not. Failing the $ActionType."
            }

            Start-Sleep -Seconds 30
        }
    }
    finally
    {
        if ($ProvisionedPSSessionCredential)
        {
            $cleanupCredential = $ProvisionedPSSessionCredential
        }
        else
        {
            $cleanupCredential = $Credential
        }

        if (![string]::IsNullOrEmpty($sessionName))
        {
            $session = Get-PSSession -ComputerName $ComputerName -Credential $cleanupCredential `
                -Name $sessionName -ErrorAction SilentlyContinue
            $session | Remove-PSSession -ErrorAction SilentlyContinue
        }

        try
        {
            $instance = Get-WSManInstance -ConnectionURI "http://$ComputerName`:5985/wsman" shell -Enumerate `
            -cred $cleanupCredential -ErrorAction SilentlyContinue | ? Name -eq $SessionName

            foreach ($instance in $instances)
            {
                Trace-Execution "Removing WSMan objects with shell ID $($instance.ShellId)"
                Remove-WSManInstance -ConnectionURI "http://$ComputerName`:5985/wsman" shell @{ShellID="$($instance.ShellId)"} `
                    -cred $cleanupCredential -ErrorAction SilentlyContinue
            }
        }
        catch
        {
            Trace-Execution "Could not clean up the WSMan instance. Exception: $_"
        }
    }
}

<#
.Synopsis
   Get the first machine in the list where a remote PSSession can be created.
#>

function Get-FirstAvailableMachine
{
    param
    (
        [Parameter(Mandatory=$true)]
        [string[]] $Machines,

        [Parameter(Mandatory=$true)]
        [PSCredential] $Credential,

        [Parameter(Mandatory = $true, ParameterSetName = "JEA")]
        [ValidateNotNullOrEmpty()]
        [string] $ConfigurationName,

        [Parameter(Mandatory = $true, ParameterSetName = "Authentication")]
        [ValidateSet("Default","Basic","Credssp","Digest","Kerberos","Negotiate","NegotiateWithImplicitCredential")]
        [string] $Authentication = "Credssp",

        [Parameter(Mandatory = $false)]
        [uint32] $Retries = 0,

        [Parameter(Mandatory = $false)]
        [uint32] $RetrySleepTimeInSeconds = 5
    )

    $RemoteSession = $null
    $found = $null
    foreach ($Machine in $Machines)
    {
        Trace-Execution "Testing remote PSSession connectivities to $Machine with user $($Credential.UserName)"
        $attempt = 0;
        try
        {
            while ($true)
            {
                $attempt++

                try
                {
                    if ($PSCmdlet.ParameterSetName -eq "JEA")
                    {
                        $session = New-PSSession -ComputerName $Machine -Credential $Credential `
                            -ConfigurationName $ConfigurationName -ErrorAction Stop
                    }
                    else
                    {
                        $session = New-PSSession -ComputerName $Machine -Credential $Credential `
                            -Authentication $Authentication -ErrorAction Stop
                    }
                    break
                }
                catch
                {
                    Trace-Warning "Test connection failed. Exception: $_"
                    if ($attempt -lt $Retries)
                    {
                        Trace-Execution "Attempt $attempt of $Retries."
                        Start-Sleep -s $RetrySleepTimeInSeconds
                        continue
                    }
                    else
                    {
                        throw "Failed to connect to $Machine after $attempt attempts. Exception: $_"
                    }
                }
            }

            $found = $Machine
            break
        }
        catch
        {
            Trace-Warning "Failed to connect to $Machine with exception $($_ | Out-String)"
        }
        finally
        {
            $session | Remove-PSSession -ErrorAction SilentlyContinue
        }
    }

    if ([string]::IsNullOrEmpty($found))
    {
        throw "Could not create a PSSession to any of the specified machines."
    }

    return $found
}

Export-ModuleMember -Function Ensure-BCDREventLog
Export-ModuleMember -Function Get-BCDRActionPlanConst
Export-ModuleMember -Function Get-BCDRActionPlanConsts
Export-ModuleMember -Function Get-FirstAvailableMachine
Export-ModuleMember -Function Start-BCDRExecutionOnRemoteComputer
Export-ModuleMember -Function Start-IdempotentBCDRRemoteExecution

# SIG # Begin signature block
# MIIoLQYJKoZIhvcNAQcCoIIoHjCCKBoCAQExDzANBglghkgBZQMEAgEFADB5Bgor
# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDUkI9CuCg+ewEj
# UjnWERSejhLSTok6F5j/AfKseK3mWKCCDXYwggX0MIID3KADAgECAhMzAAADTrU8
# esGEb+srAAAAAANOMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p
# bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI5WhcNMjQwMzE0MTg0MzI5WjB0MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
# AQDdCKiNI6IBFWuvJUmf6WdOJqZmIwYs5G7AJD5UbcL6tsC+EBPDbr36pFGo1bsU
# p53nRyFYnncoMg8FK0d8jLlw0lgexDDr7gicf2zOBFWqfv/nSLwzJFNP5W03DF/1
# 1oZ12rSFqGlm+O46cRjTDFBpMRCZZGddZlRBjivby0eI1VgTD1TvAdfBYQe82fhm
# WQkYR/lWmAK+vW/1+bO7jHaxXTNCxLIBW07F8PBjUcwFxxyfbe2mHB4h1L4U0Ofa
# +HX/aREQ7SqYZz59sXM2ySOfvYyIjnqSO80NGBaz5DvzIG88J0+BNhOu2jl6Dfcq
# jYQs1H/PMSQIK6E7lXDXSpXzAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE
# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUnMc7Zn/ukKBsBiWkwdNfsN5pdwAw
# RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW
# MBQGA1UEBRMNMjMwMDEyKzUwMDUxNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci
# tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG
# CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu
# Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0
# MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAD21v9pHoLdBSNlFAjmk
# mx4XxOZAPsVxxXbDyQv1+kGDe9XpgBnT1lXnx7JDpFMKBwAyIwdInmvhK9pGBa31
# TyeL3p7R2s0L8SABPPRJHAEk4NHpBXxHjm4TKjezAbSqqbgsy10Y7KApy+9UrKa2
# kGmsuASsk95PVm5vem7OmTs42vm0BJUU+JPQLg8Y/sdj3TtSfLYYZAaJwTAIgi7d
# hzn5hatLo7Dhz+4T+MrFd+6LUa2U3zr97QwzDthx+RP9/RZnur4inzSQsG5DCVIM
# pA1l2NWEA3KAca0tI2l6hQNYsaKL1kefdfHCrPxEry8onJjyGGv9YKoLv6AOO7Oh
# JEmbQlz/xksYG2N/JSOJ+QqYpGTEuYFYVWain7He6jgb41JbpOGKDdE/b+V2q/gX
# UgFe2gdwTpCDsvh8SMRoq1/BNXcr7iTAU38Vgr83iVtPYmFhZOVM0ULp/kKTVoir
# IpP2KCxT4OekOctt8grYnhJ16QMjmMv5o53hjNFXOxigkQWYzUO+6w50g0FAeFa8
# 5ugCCB6lXEk21FFB1FdIHpjSQf+LP/W2OV/HfhC3uTPgKbRtXo83TZYEudooyZ/A
# Vu08sibZ3MkGOJORLERNwKm2G7oqdOv4Qj8Z0JrGgMzj46NFKAxkLSpE5oHQYP1H
# tPx1lPfD7iNSbJsP6LiUHXH1MIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq
# hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x
# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv
# bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
# IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG
# EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG
# A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg
# Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
# CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03
# a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr
# rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg
# OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy
# 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9
# sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh
# dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k
# A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB
# w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn
# Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90
# lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w
# ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o
# ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD
# VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa
# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny
# bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG
# AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t
# L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV
# HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3
# dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG
# AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl
# AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb
# C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l
# hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6
# I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0
# wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560
# STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam
# ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa
# J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah
# XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA
# 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt
# Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr
# /Xmfwb1tbWrJUnMTDXpQzTGCGg0wghoJAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw
# EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN
# aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp
# Z25pbmcgUENBIDIwMTECEzMAAANOtTx6wYRv6ysAAAAAA04wDQYJYIZIAWUDBAIB
# BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO
# MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIA/pehCkQwZn7fSFbhJC5ihv
# 1vSG/qMNCVDv0o8RlW2FMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A
# cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB
# BQAEggEARc3X4DyI8BzS7ukkZReTGcdLatkO9GFedyjM+NZFVtoVIgSbD8cHcLkK
# nNBFDVjQTQIiFSSAMSHcDFZ4VyDd8QjXb0yXxSYusY6AGR4f9IbZ2rQUWFEFxkBk
# /X/Ej8XRI6zypgAenlcHWd8NmxjSqPPrWXT4/vTIh7lloqPt8DWLfWuXw544P7nf
# AmW4rNn2swr92/yippEykXXUJQf2hCdW+I7qClYI+7XcLOaGju+opgkXzNqzI5+a
# 85zmIu/ePX6yfxpKi05IK6nJgRL79z2Dn01OOhh+8xs9dMMrJTfHKtLhwrTdwuKW
# 2838cYxBexS7EHDtJNzczVoNagcg46GCF5cwgheTBgorBgEEAYI3AwMBMYIXgzCC
# F38GCSqGSIb3DQEHAqCCF3AwghdsAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFSBgsq
# hkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl
# AwQCAQUABCBo3KhQCcoROxRblYeg7C64TLpupEN31ErW9QLRVCflxwIGZQPePKi5
# GBMyMDIzMDkyMjA4MzE1OS44MTRaMASAAgH0oIHRpIHOMIHLMQswCQYDVQQGEwJV
# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE
# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1l
# cmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046QTkzNS0w
# M0UwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2Wg
# ghHtMIIHIDCCBQigAwIBAgITMwAAAdGyW0AobC7SRQABAAAB0TANBgkqhkiG9w0B
# AQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE
# BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD
# VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAeFw0yMzA1MjUxOTEy
# MThaFw0yNDAyMDExOTEyMThaMIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz
# aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv
# cnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25z
# MScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046QTkzNS0wM0UwLUQ5NDcxJTAjBgNV
# BAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggIiMA0GCSqGSIb3DQEB
# AQUAA4ICDwAwggIKAoICAQCZTNo0OeGz2XFd2gLg5nTlBm8XOpuwJIiXsMU61rwq
# 1ZKDpa443RrSG/pH8Gz6XNnFQKGnCqNCtmvoKULApwrT/s7/e1X0lNFKmj7U7X4p
# 00S0uQbW6LwSn/zWHaG2c54ZXsGY+BYfhWDgbFpCTxRzTnRCG62bkWPp6ZHbZPg4
# Ht1CRCAMhhOGTR8wI4G7wwWZwdMc6UvUUlq0ql9AxAfzkYRpi2tRvDHMdmZ3vyXp
# qhFwvRG8cgCH/TTCjW5q6aNbdqKL3BFDPzUtuCNsPXL3/E0dR2bDMqa0aNH+iIfh
# GC4/vcwuteOMCPUIDVSqDCNfIaPDEwYci1fd9gu1zVw+HEhDZM7Ea3nxIUrzt+Rf
# p5ToMMj4QAmJ6Uadm+TPbDbo8kFIK70ShmW8wn8fJk9ReQQEpTtIN43eRv9QmXy3
# Ued80osOBE+WkdMvSCFh+qgCsKdzQxQJG62cTeoU2eqNhH3oppXmyfVUwbsefQzM
# PtbinCZd0FUlmlM/dH+4OniqQyaHvrtYy3wqIafY3zeFITlVAoP9q9vF4W7KHR/u
# F0mvTpAL5NaTDN1plQS0MdjMkgzZK5gtwqOe/3rTlqBzxwa7YYp3urP5yWkTzISG
# nhNWIZOxOyQIOxZfbiIbAHbm3M8hj73KQWcCR5JavgkwUmncFHESaQf4Drqs+/1L
# 1QIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFAuO8UzF7DcH0mmsF4XQxxHQvS2jMB8G
# A1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8GA1UdHwRYMFYwVKBSoFCG
# Tmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUy
# MFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBsBggrBgEFBQcBAQRgMF4w
# XAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2Vy
# dHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3J0MAwG
# A1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwgwDgYDVR0PAQH/BAQD
# AgeAMA0GCSqGSIb3DQEBCwUAA4ICAQCbu9rTAHV24mY0qoG5eEnImz5akGXTviBw
# Kp2Y51s26w8oDrWor+m00R4/3BcDmYlUK8Nrx/auYFYidZddcUjw42QxSStmv/qW
# nCQi/2OnH32KVHQ+kMOZPABQTG1XkcnYPUOOEEor6f/3Js1uj4wjHzE4V4aumYXB
# Asr4L5KR8vKes5tFxhMkWND/O7W/RaHYwJMjMkxVosBok7V21sJAlxScEXxfJa+/
# qkqUr7CZgw3R4jCHRkPqQhMWibXPMYar/iF0ZuLB9O89DMJNhjK9BSf6iqgZoMuz
# IVt+EBoTzpv/9p4wQ6xoBCs29mkj/EIWFdc+5a30kuCQOSEOj07+WI29A4k6QIRB
# 5w+eMmZ0Jec0sSyeQB5KjxE51iYMhtlMrUKcr06nBqCsSKPYsSAITAzgssJD+Z/c
# TS7Cu35fJrWhM9NYX24uAxYLAW0ipNtWptIeV6akuZEeEV6BNtM3VTk+mAlV5/eC
# /0Y17aVSjK5/gyDoLNmrgVwv5TAaBmq/wgRRFHmW9UJ3zv8Lmk6mIoAyTpqBbuUj
# MLyrtajuSsA/m2DnKMO0Qiz1v+FSVbqM38J/PTlhCTUbFOx0kLT7Y/7+ZyrilVCz
# yAYfFIinDIjWlM85tDeU8ZfJCjFKwq3DsRxV4JY18xww8TTmod3lkr9NqGQ54Lmy
# PVc+5ibNrjCCB3EwggVZoAMCAQICEzMAAAAVxedrngKbSZkAAAAAABUwDQYJKoZI
# hvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw
# DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x
# MjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAy
# MDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIyNVowfDELMAkGA1UEBhMC
# VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV
# BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp
# bWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC
# AQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXIyjVX9gF/bErg4r25Phdg
# M/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjoYH1qUoNEt6aORmsHFPPF
# dvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1yaa8dq6z2Nr41JmTamDu6
# GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v3byNpOORj7I5LFGc6XBp
# Dco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pGve2krnopN6zL64NF50Zu
# yjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viSkR4dPf0gz3N9QZpGdc3E
# XzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYrbqgSUei/BQOj0XOmTTd0
# lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlMjgK8QmguEOqEUUbi0b1q
# GFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSLW6CmgyFdXzB0kZSU2LlQ
# +QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AFemzFER1y7435UsSFF5PA
# PBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIurQIDAQABo4IB3TCCAdkw
# EgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIEFgQUKqdS/mTEmr6CkTxG
# NSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMFwGA1UdIARV
# MFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWlj
# cm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0bTATBgNVHSUEDDAK
# BggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMC
# AYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvX
# zpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20v
# cGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYI
# KwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNydDANBgkqhkiG
# 9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv6lwUtj5OR2R4sQaTlz0x
# M7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZnOlNN3Zi6th542DYunKmC
# VgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1bSNU5HhTdSRXud2f8449
# xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4rPf5KYnDvBewVIVCs/wM
# nosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU6ZGyqVvfSaN0DLzskYDS
# PeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDFNLB62FD+CljdQDzHVG2d
# Y3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/HltEAY5aGZFrDZ+kKNxn
# GSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdUCbFpAUR+fKFhbHP+Crvs
# QWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKiexcdFYmNcP7ntdAoGokL
# jzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTmdHRbatGePu1+oDEzfbzL
# 6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZqELQdVTNYs6FwZvKhggNQ
# MIICOAIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp
# bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw
# b3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2EgT3BlcmF0aW9uczEn
# MCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOkE5MzUtMDNFMC1EOTQ3MSUwIwYDVQQD
# ExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMKAQEwBwYFKw4DAhoDFQBH
# JY2Fv+GhLQtRDR2vIzBaSv/7LKCBgzCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w
# IFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA6LeWnzAiGA8yMDIzMDkyMjA0Mjkx
# OVoYDzIwMjMwOTIzMDQyOTE5WjB3MD0GCisGAQQBhFkKBAExLzAtMAoCBQDot5af
# AgEAMAoCAQACAhb0AgH/MAcCAQACAhKDMAoCBQDouOgfAgEAMDYGCisGAQQBhFkK
# BAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEAAgMBhqAwDQYJ
# KoZIhvcNAQELBQADggEBAMMZorCacoCGWONUA0UprCJrLvLeTJ4b44I+sLVpt09M
# liZFEw+lydZaB5kgmnaYEqCJQTlzvUEwmhjJPkbJQpMwlHCO56rbfNUy6NeL+ry/
# 0k+6/jVsktnFGB/bThmMKLqURQwRSjL9B/EAZK1weD7uaWj+WfAK70+UHnykQaTb
# qKDjRh8/PmFyzQ+v7STwmuB1DGuZySfYfvL9TSUrf9HkzF7zW/Nsr22psW7w9FfI
# ueH7ZDfksQsYTJuUWooeJf+V52ZFUo/l6hi82XIFVnj0H35uMoroyH2K52hRA72m
# yhrZs64BZ33zNaxyAkj/TR0oLZORGQvAac7WrnibMxUxggQNMIIECQIBATCBkzB8
# MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVk
# bW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1N
# aWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAdGyW0AobC7SRQABAAAB
# 0TANBglghkgBZQMEAgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEE
# MC8GCSqGSIb3DQEJBDEiBCA3rNMTxzFofnLdIXdhnDLVm7Z4CazIOtTLtIFxJE8r
# RTCB+gYLKoZIhvcNAQkQAi8xgeowgecwgeQwgb0EIMy8YXkCALv57c5sRhrPTub1
# q4TwJ6oVA36k8IiI/AcMMIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgT
# Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m
# dCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENB
# IDIwMTACEzMAAAHRsltAKGwu0kUAAQAAAdEwIgQgvPNc74y+0hlzLzQQhN//sdFX
# fKbYz0wCrq96EjDDf4AwDQYJKoZIhvcNAQELBQAEggIAfTq7lLZD21ivy2u44hzL
# WqVIBWeQAAOwNk5nVrjoR9w5v7oVbJdJKLSqpYcS3NYRV2r6zGfY8BmuGJJJnMvV
# sKa4FYcjmbYfGCbGhBAX/Ugr80KpyJg+Eszod91Do7pnG1fsaTkdOpP3ov51ELx/
# cl8EDS4VBVQk6q3Ot2QIL9KNgqetj/2wJZKvngvQvSoFCL7wNNxCPmuI85m4od/2
# z8Sp2PTweFjSPTRJR3Du4dU7s8EYNVhw3P/EwWeLCR04akFu5najgzqXdmUAuQGK
# jjncVo+b/4fmyTBuglxqOg+ufmwtmZDEQ9cCu46uEK4wqf6q9fOyCr77wjUF9b0f
# lE4Ra9bGhUlD1UAiJTn5qlQ7bwp0MdQeFe1E7Y3WGb3wE0rnZpdRk+EwLawJAWn0
# X421QvOqEA7vOftmx4Tmt4I6tQBHjq506LO1ShLuKXpK29iLuDQL4MbDJtb11sZW
# ySN7VtiJaTXD3rp7HvJD7dANNUWt4Un1Yj8plM/oN70IDDQluLjGL//8EKHuQfBG
# mLPk9jp6gjINTEb7u4nJJtknarCOwsiHciV413Rz6Neoa6AVrj7XIvhEIzWcQdaz
# 6BOb1y3utpNRwB0A5UuOQjP+6LIvy4lmUyUNeGpBVmOsdxtgh/Pfp9MnVl5Dfb+u
# taeG3RajzlQmHK2tUNBRpvI=
# SIG # End signature block