AzureStackVaaS

1.0.3

Automation/LaunchVaaSTests.ps1

##-----------------------------------------------------------------------
## <copyright file="LaunchVaaSTests.ps1" company="Microsoft Corp.">
## Copyright (c) Microsoft Corp. All rights reserved.
## </copyright>
##-----------------------------------------------------------------------

<#
.SYNOPSIS
 
Launches a VaaS Test Pass based off of the supplied parameters.
 
.DESCRIPTION
 
Launches a VaaS Test Pass based off of the supplied parameters using the AzureStackVaaS PowerShell cmdlets.
 
.PARAMETER VaaSSolutionName
 
VaaSSolutionName
 
.PARAMETER VaaSTestPassName
 
VaaSTestPassName
 
.PARAMETER VaaSTestCategory
 
VaaSTestCategory - Integration, Functional, or Reliability
 
.PARAMETER VaaSOnPremAgentName
 
Name of the agent for executing tests on premise. If not specified, the local machine name will be used.
 
.PARAMETER VaaSAccountCreds
 
VaaSAccountCreds - PSCredential type
 
.PARAMETER VaaSApplicationId
 
VaaSApplicationId
 
.PARAMETER VaaSAccountTenantId
 
VaaSAccountTenantId
 
.PARAMETER VaaSPackageUri
 
Optional. Overrides default VaaS package storage location
 
.PARAMETER VaaSPortalUri
 
Optional. Specifies VaaS Portal Endpoint.
 
.PARAMETER VaaSApplicationUri
 
Optional. Specifies Application Endpoint.
 
.PARAMETER VaaSBaseServiceUri
 
Optional. Specifies VaaS Service Endpoint.
 
.PARAMETER VaaSServiceResourceId
 
Optional. Specifies VaaS Service Resource Id.
 
.PARAMETER AadTenantId
 
AadTenantId
 
.PARAMETER ServiceAdminUser
 
ServiceAdminUser
 
.PARAMETER ServiceAdminPassword
 
ServiceAdminPassword
 
.PARAMETER TenantAdminUser
 
TenantAdminUser
 
.PARAMETER TenantAdminPassword
 
TenantAdminPassword
 
.PARAMETER ExternalFqdn
 
External FQDN for the environment
 
.PARAMETER LogFilePath
 
Path to log files from tests. If the DoNotWait switch is specified, use cmdlet Get-AzureStackVaaSTestLaunchLogs
to obtain log files.
 
.PARAMETER MaxDurationInHrs
 
Specifies how long execution of this script should take before cancelling any pending tests and returning
the failure exit code (0)
 
.PARAMETER DoNotWait
 
The DoNotWait switch bypasses the test execution monitor loop that normally executes once tests are scheduled.
 
.PARAMETER UseVaaSSPNAuth
 
The UseVaaSSPNAuth switch specifies that SPN credentials are supplied via parameter VaaSAccountCreds.
Note that parameters VaaSApplicationId and VaaSApplicationUri are not applicable when specifying the UseVaaSSPNAuth switch.
 
.EXAMPLE
 
Create account credentials object:
 
$VaaSAccountCreds = New-Object System.Management.Automation.PSCredential (“vaasUserId@vaasTenant.onmicrosoft.com”, ConvertTo-SecureString “Password” -AsPlainText -Force)
 
Launch VaaS Test Pass using account credentials:
 
.\LaunchVaaSTests.ps1 -VaaSTestCategory Functional -VaaSAccountCreds $VaaSAccountCreds -VaaSAccountTenantId <VaaSAccountTenantId>
-AadTenantId <AadTenantId> -ServiceAdminUser <ServiceAdminUser> -ServiceAdminPassword <ServiceAdminPassword>
-TenantAdminUser <TenantAdminUser> -TenantAdminPassword <TenantAdminPassword> -ExternalFqdn <ExternalFqdn> -MaxDurationInHrs 48
 
.LINK
 
https://www.powershellgallery.com/packages/AzureStackVaaS
 
#>

param(
    [Parameter(Mandatory=$false)][string]$VaaSSolutionName=$null,
    [Parameter(Mandatory=$false)][string]$VaaSTestPassName=$null,
    [Parameter(Mandatory=$true)][ValidateSet("Integration","Functional","Reliability")][string]$VaaSTestCategory,
    [Parameter(Mandatory=$false)][string]$VaaSOnPremAgentName=$null,
    [Parameter(Mandatory=$true)][ValidateNotNull()][PSCredential]$VaaSAccountCreds,
    [Parameter(Mandatory=$false)][string]$VaaSApplicationId=$null,
    [Parameter(Mandatory=$true)][ValidateNotNull()][string]$VaaSAccountTenantId,
    [Parameter(Mandatory=$false)][string]$VaaSTestManifestSource=$null,
    [Parameter(Mandatory=$false)][string]$VaaSPackageUri=$null,
    [Parameter(Mandatory=$false)][string]$VaaSPortalUri=$null,
    [Parameter(Mandatory=$false)][string]$VaaSApplicationUri=$null,
    [Parameter(Mandatory=$false)][string]$VaaSBaseServiceUri=$null,
    [Parameter(Mandatory=$false)][string]$VaaSServiceResourceId=$null,
    [Parameter(Mandatory=$true)][ValidateNotNull()][string]$AadTenantId,
    [Parameter(Mandatory=$true)][ValidateNotNull()][string]$ServiceAdminUser,
    [Parameter(Mandatory=$true)][ValidateNotNull()][string]$ServiceAdminPassword,
    [Parameter(Mandatory=$true)][ValidateNotNull()][string]$TenantAdminUser,
    [Parameter(Mandatory=$true)][ValidateNotNull()][string]$TenantAdminPassword,
    [Parameter(Mandatory=$true)][string]$ExternalFqdn,
    [Parameter(Mandatory=$false)][string]$LogFilePath=$null,
    [Parameter(Mandatory=$true)][int]$MaxDurationInHrs,
    [Parameter(Mandatory=$false)][switch]$DoNotWait,
    [Parameter(Mandatory=$false)][switch]$UseVaaSSPNAuth
)

[System.Reflection.Assembly]::LoadWithPartialName("mscorlib") | Out-Null

try
{
Add-Type -TypeDefinition @"
public struct LaunchData
{
public System.DateTime durationTimeout;
public string[] logPaths;
public string onPremAgentDirectory;
public int testResult;
public int titleCounter;
public string vaaSPackageDirectory;
}
"@
}
catch
{
    Write-Host "Add-Type may already be added for this PowerShell Session ..."
}
finally
{
$launcherData = New-Object LaunchData
}

Add-Type -TypeDefinition @"
public struct EnvInfo
{
public string DeploymentID;
public string Version;
public string Prefix;
public string CompanyName;
public string ServerSku;
public string Topology;
public string Timezone;
public string HardwareOEM;
public string RegionName;
public string DomainNetBIOSName;
public string DomainFQDN;
public string Timeserver;
public int NumberOfNodes;
}
"@
$envInfo = New-Object EnvInfo

New-Variable -Name BaseURIVaaSStorage -Value "https://vaastestpacksprodeastus.blob.core.windows.net" -Option Constant
New-Variable -Name BaseURIVaaSPortal -Value "https://azurestackvalidation.com" -Option Constant
New-Variable -Name GetStampInfoUri -Value "https://ASAppGateway:4443/ServiceTypeId/4dde37cc-6ee0-4d75-9444-7061e156507f/CloudDefinition/GetStampInformation" -Option Constant
New-Variable -Name IsProductionRun -Value $true

function CreateVaaSDirectory
{
    if (!(Test-Path -Path "$($launcherData.vaaSPackageDirectory)" -PathType Container))
    {
        Write-Verbose "Creating vaaSPackageDirectory ..."
        New-Item -ItemType Directory -Path $launcherData.vaaSPackageDirectory | Out-Null
    }
    else
    {
        Write-Warning "Path $($launcherData.vaaSPackageDirectory) already exists ..."
    }
}

function CreateVaaSLogDirectory
{
    if (!(Test-Path -Path "$LogFilePath" -PathType Container))
    {
        Write-Verbose "Creating VaaS Log Directory ..."
        New-Item -ItemType Directory -Path $LogFilePath | Out-Null 
    }
}

function CreateVaaSSolution
{
    DisplayTitle -Title "Creating VaaS Solution"
    Write-Host "Creating VaaS Solution with name $VaaSSolutionName ..."
    return (New-AzureStackVaaSSolution -Name $VaaSSolutionName -Force -ErrorAction Stop)
}

function CreateVaaSTestPass
{
    DisplayTitle -Title "Creating VaaS Test Pass"
    Write-Host "Creating VaaS Test Pass with name $VaaSTestPassName ..."
    $tpParams = @{'SolutionName' = $VaaSSolutionName;'Name' = $VaaSTestPassName}

    if($envInfo.Version -ne "")
    {
        $tpParams.Add('Build',$envInfo.Version)
    }

    if($envInfo.ServerSku -ne "")
    {
        $tpParams.Add('ServerSku',$envInfo.ServerSku)
    }
    
    if($envInfo.Topology -ne "")
    {
        $tpParams.Add('Topology',$envInfo.Topology)
    }
    
    if($envInfo.RegionName -ne "")
    {
        $tpParams.Add('RegionName',$envInfo.RegionName)
    }
    
    if($envInfo.NumberOfNodes -ne "")
    {
        $tpParams.Add('NumberOfNodes',$envInfo.NumberOfNodes)
    }

    return (New-AzureStackVaaSTestPass @tpParams -Force -ErrorAction Stop)
}

function InstallVaaSPSModules
{
    DisplayTitle -Title "Installing Azure Stack VaaS PowerShell Module"
    
    if($VaaSPackageUri -eq "$BaseURIVaaSStorage\packages")
    {
        Write-Host "Installing from PowerShell Gallery ..."
        Install-Module -Name AzureStackVaaS -Scope AllUsers -AllowClobber -Force -ErrorVariable InstallError -ErrorAction SilentlyContinue
        if($InstallError)
        {
            TerminateScript "Couldn't install Azure Stack VaaS PowerShell module..."
        }
        return
    }    
    
    $vaasPSModulesUri = "$VaaSPackageUri/AzureStackVaaS.zip"
    Write-Host "Downloading latest Azure Stack VaaS PowerShell Module from:"
    Write-Host $vaasPSModulesUri
    try
    {
        Invoke-WebRequest -Uri $vaasPSModulesUri -outfile "$($launcherData.vaaSPackageDirectory)\AzureStackVaaS.zip"
    }
    catch
    {
        Write-Warning "Unable to download file from $vaasPSModulesUri"
        Write-Warning "Attempting retry in 30 seconds ..."
        Start-Sleep -s 30 | Out-Null
        try
        {
            Invoke-WebRequest -Uri $vaasPSModulesUri -outfile "$($launcherData.vaaSPackageDirectory)\AzureStackVaaS.zip"
        }
        catch
        {
            Write-Warning "Unable to download file on retry from:`r`n$vaasPSModulesUri"
            return
        }
    }
    
    $psModulePath = $env:ProgramFiles + '\WindowsPowerShell\Modules'    
    Write-Host "Extracting latest Azure Stack VaaS PowerShell Module to:`r`n$psModulePath"
    Expand-Archive -Path "$($launcherData.vaaSPackageDirectory)\AzureStackVaaS.zip" -DestinationPath "$psModulePath" -Force -ErrorAction SilentlyContinue -ErrorVariable ExpandError
    if($ExpandError)
    {
        if($ExpandError[0].CategoryInfo.Reason -imatch 'UnauthorizedAccessException')
        {
            Write-Warning "A PowerShell lock currently exists on the Azure Stack VaaS PowerShell."
            Write-Warning "To update binaries, please close the applicable PowerShell window or ISE."
        }        
    }
}

function InstallVaaSOnPremAgent
{
    DisplayTitle -Title "Installing VaaS OnPrem Agent"
    Write-Host "Stopping any running VaaS OnPrem Agent(s) ..."
    StopVaaSOnPremAgentProcess    
    Write-Host "Downloading latest VaaS OnPrem Agent from:"
    $vaasOnPremAgentUri = "$VaaSPackageUri/Microsoft.VaaSOnPrem.TaskEngineHost.1.0.3.nupkg"
    Write-Host $vaasOnPremAgentUri
    New-Item -ItemType Directory -Force -Path $launcherData.vaaSPackageDirectory | Out-Null
    try
    {
        Invoke-WebRequest -Uri $vaasOnPremAgentUri -outfile "$($launcherData.vaaSPackageDirectory)\OnPremAgent.zip"
    }
    catch
    {
        Write-Warning "Unable to download file from $vaasOnPremAgentUri"
        Write-Warning "Attempting retry in 30 seconds ..."
        Start-Sleep -s 30 | Out-Null
        Invoke-WebRequest -Uri $vaasOnPremAgentUri -outfile "$($launcherData.vaaSPackageDirectory)\OnPremAgent.zip"
    }
    
    Write-Host "Extracting latest VaaS OnPrem Agent to:"
    Write-Host $launcherData.vaaSPackageDirectory
    Expand-Archive -Path "$($launcherData.vaaSPackageDirectory)\OnPremAgent.zip" -DestinationPath "$($launcherData.vaaSPackageDirectory)\OnPremAgent" -Force
}

function InstallVaaSPreReq
{
    DisplayTitle -Title "Installing VaaS Prerequisites"
    Push-Location $launcherData.onPremAgentDirectory
    $result = @()
    Invoke-Expression -Command '.\VaaSPreReqInstall.ps1 -VaaSPackageUri $VaaSPackageUri -AadTenantId $AadTenantId' -OutVariable +result
    if($result[$result.count-1] -ne 0)
    {
      Pop-Location
      Write-Error "VaaS Prerequisite installation failed. Check logs for details."             
      throw [System.AggregateException] "Unable to install VaaS Prerequisites ..."        
    }
    Write-Host "VaaS Prerequisite installation succeeded. Check logs for details."
    Pop-Location
}

function InitializeLaunchData
{
    $launcherData.vaaSPackageDirectory = $env:Temp + "\VaaS"
    $launcherData.onPremAgentDirectory = $launcherData.vaaSPackageDirectory + "\OnPremAgent\lib\net46"
    $launcherData.durationTimeout = (Get-Date).AddHours($MaxDurationInHrs)
    $launcherData.logPaths = @()
    $launcherData.testResult = 0
    $launcherData.titleCounter = 0
}

function ValidateAndResolveAuthMethodParameters
{
    if(($VaaSPortalUri -eq "") -and ($VaaSBaseServiceUri -eq "") -and ($VaaSServiceResourceId -eq ""))
    {
        $script:IsProductionRun = $true
    }
    elseif(($VaaSPortalUri -ne "") -and ($VaaSBaseServiceUri -ne "") -and ($VaaSServiceResourceId -ne ""))
    {
        $script:IsProductionRun = $false
    }
    else
    {
        throw [System.ArgumentException] "Parameters VaaSPortalUri, VaaSBaseServiceUri, and VaaSServiceResourceId must be either all specified, or none specified ..."
    }    
    
    if(($IsProductionRun) -or ($UseVaaSSPNAuth))
    {
        if(($VaaSApplicationId -ne "") -or ($VaaSApplicationUri -ne ""))
        {
            throw [System.ArgumentException] "Parameters VaaSApplicationId and/or VaaSApplicationUri should not be specified when launching a production level run or when using SPN credentials ..."
        }    
    }
    elseif(!$UseVaaSSPNAuth)
    {
        Write-Host "Verifying that parameters VaaSApplicationId and VaaSApplicationUri are present as`r`nthe UseVaaSSPNAuth switch was not specified..."
        if(($VaaSApplicationId -eq "") -or ($VaaSApplicationUri -eq ""))
        {
            throw [System.ArgumentException] "Parameters VaaSApplicationId and VaaSApplicationUri must both be specified when using non-SPN credentials are supplied ..."
        }    
    }
}

function ValidateAndResolveTestParameters
{
    $vaasGroupManifest = "$PSScriptRoot\TestGroupManifests\$VaaSTestCategory.json"
    if (!(Test-Path -Path "$vaasGroupManifest" -PathType Leaf))
    {
        throw [System.IO.FileNotFoundException] "The required test manifest file for executing $VaaSTestCategory tests was not found at location:`r`n$vaasGroupManifest"
    }
    
    $ReportingScriptPath = "$PSScriptRoot\Generate-VaaSReport.ps1"
    if (!(Test-Path -Path "$ReportingScriptPath" -PathType Leaf))
    {
        throw [System.IO.FileNotFoundException] "VaaS test pass report generator script was not found at location:`r`n$ReportingScriptPath`r`nPlease ensure this file exists at the expected location and retry ..."
    }

    $ReportingStylePath = "$PSScriptRoot\VaaSStyle.css"
    if (!(Test-Path -Path "$ReportingStylePath" -PathType Leaf))
    {
        throw [System.IO.FileNotFoundException] "VaaS test pass report generator style sheet was not found at location:`r`n$ReportingStylePath`r`nPlease ensure this file exists at the expected location and retry ..."
    }

    if($VaaSTestManifestSource -eq "")
    {
        $script:VaaSTestManifestSource = $BaseURIVaaSStorage
        Write-Host "VaaS Test Manifest source updated to:`r`n$VaaSTestManifestSource"
    }
    
    if($VaaSPackageUri -eq "")
    {
        $script:VaaSPackageUri = "$BaseURIVaaSStorage/packages"
        Write-Host "VaaS Storage endpoint updated to:`r`n$VaaSPackageUri"
    }
    
    if($VaaSPortalUri -eq "")
    {
        $script:VaaSPortalUri = $BaseURIVaaSPortal 
        Write-Host "VaaS Portal endpoint updated to:`r`n$VaaSPortalUri"
    }
}

function ValidateAndResolveEnvironmentParameters
{
    if($VaaSOnPremAgentName -eq "")
    {
        $script:VaaSOnPremAgentName = (Get-WmiObject win32_computersystem).DNSHostName + "." + (Get-WmiObject win32_computersystem).Domain        
        Write-Host "Defaulting to VaaS OnPrem agent name $VaaSOnPremAgentName, as an agent name was not provided ..."
    }
    
    GetStampInformation

    if($VaaSTestPassName -eq "")
    {
        $script:VaaSTestPassName = "TP" + $(Get-Date -Format yyMMddHHmm)
        Write-Host "Defaulting to VaaS test pass name $VaaSTestPassName, as a test pass name was not provided ..."
    }
    
    if($VaaSSolutionName -eq "")
    {
        $script:VaaSSolutionName = "AzureStack-VaaS"
        Write-Host "Defaulting to VaaS solution name $VaaSSolutionName, as a solution name was not provided ..."
    }
}

function ValidateAndResolveParameters
{    
    DisplayTitle -Title "Validating and Resolving Parameters"
    ValidateAndResolveAuthMethodParameters
    ValidateAndResolveTestParameters    
    ValidateAndResolveEnvironmentParameters    
}

function ConfigureGroupManifestFile
{
    DisplayTitle -Title "Configuring VaaS Group Manifest File"

    $vaasGroupManifest = "$PSScriptRoot\TestGroupManifests\$VaaSTestCategory.json"
    $vaasGroupManifestData = Get-Content $vaasGroupManifest
    
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[VaaSTestManifestSource]", $VaaSTestManifestSource)
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[ExternalFqdn]", $ExternalFqdn)
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[TenantId]", $AadTenantId)
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[AdminArmEndpoint]", "https://adminmanagement.$ExternalFqdn/")
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[TenantArmEndpoint]", "https://management.$ExternalFqdn/")
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[ServiceAdminUser]", $ServiceAdminUser)
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[ServiceAdminPassword]", $ServiceAdminPassword)
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[TenantAdminUser]", $TenantAdminUser)
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[TenantAdminPassword]", $TenantAdminPassword)
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[NumberOfNodes]", $envInfo.NumberOfNodes.ToString())    
    $vaasGroupManifestData = $vaasGroupManifestData.Replace("[RegionName]", $envInfo.RegionName)    

    $vaasGroupManifest = $vaasGroupManifest.Replace(".json", "_Resolved.json")
    $vaasGroupManifestData | Set-Content $vaasGroupManifest -Force
    Write-Host "Group Manifest $vaasGroupManifest created ..."
    return $vaasGroupManifest
}

function CancelActiveTests
{
    param(
    [parameter(Mandatory=$true)][Object[]]$testLaunchManifests
    )

    foreach($testLaunchManifest in $testLaunchManifests)
    {        
        $testManifest = Get-AzureStackVaaSTestLaunch -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName -Name $testLaunchManifest.TestLaunchName
        if (($testManifest.State -eq 'Accepted') -or ($testManifest.State -eq 'Running'))
        {
            Stop-AzureStackVaaSTestLaunch -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName -Name $testLaunchManifest.TestLaunchName -ErrorAction Ignore -ErrorVariable StopTestError

              if($StopTestError)
            {
                Write-Warning "Unable to Stop Test $($testLaunchManifest.TestLaunchName) due to $($StopTestError[0].CategoryInfo.Reason). Retrying ..."
                Start-Sleep -s 5 | Out-Null
                Stop-AzureStackVaaSTestLaunch -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName -Name $testLaunchManifest.TestLaunchName -ErrorAction Ignore
            }
        }
    }
}

function StartVaaSOnPremAgent
{
    DisplayTitle -Title "Starting VaaS OnPrem Agent"

    Write-Host "Verifying that 'login.windows.net' can be resolved ..."
    Resolve-DnsName login.windows.net -ErrorVariable ResolutionError -ErrorAction SilentlyContinue | Out-Null
    if($ResolutionError)
    {
        Write-Error "Terminating script as 'login.windows.net' cannot be resolved ..."
        throw $ResolutionError[0].Exception
    }
    
    $agentArgs = @("-t $VaaSAccountTenantId","-p $($VaaSAccountCreds.GetNetworkCredential().password)","-n $VaaSOnPremAgentName")
    if($VaaSApplicationId -eq "")
    {
        $agentArgs += "-c $(($VaaSAccountCreds).UserName)"
    }
    else
    {
        $agentArgs += "-u $(($VaaSAccountCreds).UserName)"
        $agentArgs += "-c $VaaSApplicationId"
    }
    
    if($VaaSBaseServiceUri -ne "")
    {
        $agentArgs += "-l $VaaSBaseServiceUri"
    }
    
    if($VaaSServiceResourceId -ne "")
    {
        $agentArgs += "-r $VaaSServiceResourceId"
    }
    
    if($VaaSApplicationUri -ne "")
    {
        $agentArgs += "-z $VaaSApplicationUri"
    }
    
    Write-Host "Checking to see if Agent $VaaSOnPremAgentName already exists ..."
    $agentInstance = Get-AzureStackVaaSOnPremAgent -Name $VaaSOnPremAgentName -ErrorAction SilentlyContinue -ErrorVariable GetAgentError | Out-Null
    if($GetAgentError)
    {
        Write-Host "Did not find Agent $VaaSOnPremAgentName. Starting agent ..."
    }
    else
    {
        RemoveVaaSOnPremAgent -agentName $VaaSOnPremAgentName
    }    
    
    Push-Location $launcherData.onPremAgentDirectory
    Start-Process -FilePath "Microsoft.VaaSOnPrem.TaskEngineHost.exe" -ArgumentList $agentArgs -WindowStyle Minimized -Verb runas
    Pop-Location
    Write-Host "VaaS OnPrem Agent started ..."
    Write-Host "Waiting for OnPrem Agent to register ..."    
    Write-Host "Verifying OnPrem Agent status ..."
    VerifyVaaSOnPremAgentStatus -agentName $VaaSOnPremAgentName
    return $VaaSOnPremAgentName
}

function VerifyVaaSOnPremAgentStatus
{
    param(
    [parameter(Mandatory=$true)][ValidateNotNull()][string]$agentName
    )
    
    Start-Sleep -s 10 | Out-Null
    $agentInstance = Get-AzureStackVaaSOnPremAgent -Name $agentName -ErrorAction Ignore -ErrorVariable GetAgentError
    if($GetAgentError)
    {
        Write-Verbose "Unable to get agent $VaaSOnPremAgentName, retrying after 30 seconds ..."
        Start-Sleep -s 30 | Out-Null
        $agentInstance = Get-AzureStackVaaSOnPremAgent -Name $agentName -ErrorAction Ignore -ErrorVariable GetAgentError        
        if($GetAgentError)
        {
            $launcherData.testResult = 1
            $launcherData.testResult
            ShutdownVaaSOnPremAgent
            RemoveVaaSOnPremAgent
            exit $launcherData.testResult
        }        
    }

    if($agentInstance.ApproximatePendingTests -gt 0)
    {
        Write-Warning "Agent has $($agentInstance.ApproximatePendingTests) pending tests ..."
    }
    
    Write-Host "Agent ready for launching tests ..."
}

function ShutdownVaaSOnPremAgent
{
    $agentInstance = Get-AzureStackVaaSOnPremAgent -Name $VaaSOnPremAgentName -ErrorAction Ignore -ErrorVariable ShutdownAgentError
    if($ShutdownAgentError)
    {
        Write-Verbose "Unable to get agent $VaaSOnPremAgentName"
        return
    }
        
    [DateTime]$shutdownTimeout = (Get-Date).AddMinutes(5)
    while($agentInstance.ApproximatePendingTests -gt 0)
    {
        if((Get-Date) -gt $launcherData.durationTimeout)
        {
            Write-Warning "Tests are still being processed by agent after 5 minutes, forcing Agent shutdown ..."
            break
        }    

        Start-Sleep -Seconds 20 | Out-Null
        $agentInstance = Get-AzureStackVaaSOnPremAgent -Name $VaaSOnPremAgentName
        Write-Host "Waiting for $($agentInstance.ApproximatePendingTests) tests to complete before shutting down Agent ..."
    }

    StopVaaSOnPremAgentProcess
}

function StopVaaSOnPremAgentProcess
{
    $process = Get-Process -ProcessName "Microsoft.VaaSOnPrem.TaskEngineHost" -ErrorAction Ignore
    foreach ($proc in $process)
    { 
        Stop-Process $proc -Force -ErrorAction Ignore
    }
}

function RemoveVaaSOnPremAgent
{
    param(
    [parameter(Mandatory=$true)][ValidateNotNull()][string]$agentName
    )

    Write-Host "Attempting to remove Agent $agentName ..."
    Remove-AzureStackVaaSOnPremAgent -Name $agentName -ErrorAction SilentlyContinue -ErrorVariable RemoveAgentError -Force | Out-Null
    if($RemoveAgentError)
    {
        Write-Warning "Unable to remove Agent $agentName due to ..."
        $errorMsg = $RemoveAgentError.Exception.InnerException.Message.ToString()
        Write-Warning $errorMsg
    }
    else
    {
        Write-Host "Agent $agentName was removed successfully ..."
    }
}

function GetStampInformation
{
    try
    {
        Write-Host "Attempting to get stamp information via:`r`n$GetStampInfoUri"
        $params = Invoke-RestMethod -Method Get -Uri $GetStampInfoUri
    }
    catch
    {
        Write-Warning "Unable to get stamp infomation. Retrying in 15 seconds ..."
        Start-Sleep -Seconds 15
        try
        {
            $params = Invoke-RestMethod -Method Get -Uri $GetStampInfoUri
        }
        catch
        {
            Write-Warning "Unable to get stamp infomation on retry"
        }
    }
    finally
    {
        if($params)
        {
            $envInfo.DeploymentID = $params.DeploymentID
            $envInfo.Version = $params.Version
            $envInfo.Prefix = $params.Prefix
            $envInfo.CompanyName = $params.CompanyName
            $envInfo.ServerSku = $params.ServerSku
            $envInfo.Topology = $params.Topology
            $envInfo.Timezone = $params.Timezone
            $envInfo.HardwareOEM = $params.HardwareOEM
            $envInfo.RegionName = $params.RegionName
            $envInfo.DomainNetBIOSName = $params.DomainNetBIOSName
            $envInfo.DomainFQDN = $params.DomainFQDN
            $envInfo.Timeserver = $params.Timeserver
            $envInfo.NumberOfNodes = $params.NumberOfNodes
        }
        
        if($envInfo.NumberOfNodes -eq "")
        {
            Write-Host "Defaulting Number Of Nodes to 4 as environment information is not available ..."
            $envInfo.NumberOfNodes = 4
        }
    }
}

function GenerateVaaSReport
{
    Write-Host "Generating VaaS Report"
    if($IsProductionRun)
    {
        Write-Host "Production Run ..."
        if($UseVaaSSPNAuth)
        {
            Write-Host "Using SPN Authentication"
            . $PSScriptRoot\Generate-VaaSReport.ps1 -VaaSAccountCreds $VaaSAccountCreds -UseVaaSSPNAuth -VaaSAccountTenantId $VaaSAccountTenantId -VaaSSolutionName $VaaSSolutionName -VaaSTestPassName $VaaSTestPassName -VaaSPackageUri $VaaSPackageUri -LogFilePath $LogFilePath -ProcessTestLogsFromPath $LogFilePath
        }
        else
        {
            Write-Host "Using User Authentication"
            . $PSScriptRoot\Generate-VaaSReport.ps1 -VaaSAccountCreds $VaaSAccountCreds -VaaSApplicationId $VaaSApplicationId -VaaSApplicationUri $VaaSApplicationUri -VaaSAccountTenantId $VaaSAccountTenantId -VaaSSolutionName $VaaSSolutionName -VaaSTestPassName $VaaSTestPassName -VaaSPackageUri $VaaSPackageUri -LogFilePath $LogFilePath -ProcessTestLogsFromPath $LogFilePath
        }            
    }
    else
    {
        Write-Host "Non-Production Run ..."
        if($UseVaaSSPNAuth)
        {
            Write-Host "Using SPN Authentication"
            . $PSScriptRoot\Generate-VaaSReport.ps1 -VaaSAccountCreds $VaaSAccountCreds -UseVaaSSPNAuth -VaaSAccountTenantId $VaaSAccountTenantId -VaaSSolutionName $VaaSSolutionName -VaaSTestPassName $VaaSTestPassName -VaaSPackageUri $VaaSPackageUri -LogFilePath $LogFilePath -ProcessTestLogsFromPath $LogFilePath -VaaSPortalUri $VaaSPortalUri -VaaSBaseServiceUri $VaaSBaseServiceUri -VaaSServiceResourceId $VaaSServiceResourceId
        }
        else
        {
            Write-Host "Using User Authentication"
            . $PSScriptRoot\Generate-VaaSReport.ps1 -VaaSAccountCreds $VaaSAccountCreds -VaaSApplicationId $VaaSApplicationId -VaaSApplicationUri $VaaSApplicationUri -VaaSAccountTenantId $VaaSAccountTenantId -VaaSSolutionName $VaaSSolutionName -VaaSTestPassName $VaaSTestPassName -VaaSPackageUri $VaaSPackageUri -LogFilePath $LogFilePath -ProcessTestLogsFromPath $LogFilePath -VaaSPortalUri $VaaSPortalUri -VaaSBaseServiceUri $VaaSBaseServiceUri -VaaSServiceResourceId $VaaSServiceResourceId
        }
    }
}

function DownloadLogs
{
    param(
    [parameter(Mandatory=$true)][string]$testLaunchName,
    [parameter(Mandatory=$true)][string]$testName
    )

    [string]$logFullName = $LogFilePath + "\" + $testLaunchName + ".zip"
    Start-Sleep -s 2  | Out-Null
    Get-AzureStackVaaSTestLaunchLogs -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName -TestLaunchName $testLaunchName -Path $logFullName -ErrorAction SilentlyContinue -ErrorVariable downloadError

    if($downloadError)
    {
        if($downloadError[0].CategoryInfo.Reason -imatch 'AggregateException')
        {
            Write-Warning "Unable to download logs for test $testLaunchName to $logFullName"
            Write-Warning "Command used to download logs:"
            Write-Warning "Get-AzureStackVaaSTestLaunchLogs -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName -TestLaunchName $testLaunchName -Path $logFullName"
            Write-Host ""
            Start-Sleep -s 2  | Out-Null
            return
        }        
    }

    $launcherData.logPaths += "Logs for test $testName [$testLaunchName] downloaded to $logFullName"
    Write-Host $launcherData.logPaths[-1]
    Write-Host ""
}

function TestMonitor
{
    param(
    [parameter(Mandatory=$false)][int]$refreshRate=20,
    [parameter(Mandatory=$false)][int]$refreshInterval=600,
    [parameter(Mandatory=$true)][Object[]]$testLaunchManifests
    )
    
    DisplayTitle -title "Test Pass Monitor"
    Start-Sleep -Seconds 2  | Out-Null
    
    $prevTestManifests = $null
    $nextInterval = (Get-Date).AddSeconds(-1)
    while($true)
    {
        if((Get-Date) -gt $launcherData.durationTimeout)
        {
            $launcherData.testResult = 1
            CancelActiveTests -testLaunchManifests $testLaunchManifests
            return        
        }

        $testManifests = @()
        foreach($testLaunchManifest in $testLaunchManifests)
        {        
            $testManifests += Get-AzureStackVaaSTestLaunch -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName -Name $testLaunchManifest.TestLaunchName -ErrorAction SilentlyContinue -ErrorVariable GetTestError
            if($getTestError)
            {
                Write-Warning "Could not get information for test $($testLaunchManifest.TestLaunchName); Reason - $($GetTestError[0].CategoryInfo.Reason)"
            }
        }

        $testPassActive = $false
        foreach ($testManifest in $testManifests)
        {
            if (($testManifest.State -eq 'Accepted') -or ($testManifest.State -eq 'Running'))
            {
                $testPassActive = $true
                break
            }
        }

        # Check to see if there are any updates
        if($prevTestManifests)
        {
            foreach ($prevTestManifest in $prevTestManifests)
            {
                #output table, reset nextInterval, and continue
                $testManifest = $testManifests | where {$_.TestLaunchName -eq $prevTestManifest.TestLaunchName}
                if($prevTestManifest.State -ne $testManifest.State)
                {
                    Write-Host "[$(Get-Date -Format 'MM-dd-yy hh:mm:ss')] Checking Test Pass Status ..."
                    Write-Host "Test Pass URL $VaaSPortalUri/TestPass/Manage/Summary?solutionName=$VaaSSolutionName&testPassName=$VaaSTestPassName"
                    Find-AzureStackVaaSTestLaunch -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName | ft @{Expression={$_.TestName};Label="Test Name"},@{Expression={$_.CreatedTimeUtc};Label="Created"},@{Expression={if($_.StartedTimeUtc -eq "1/1/0001 12:00:00 AM"){""}else{$_.StartedTimeUtc}};Label="Started"},State,AgentName
                    $nextInterval = (Get-Date).AddSeconds($refreshInterval)
                    $prevTestManifests = $testManifests

                    if( ($testManifest.State -eq "Succeeded") -or ($testManifest.State -eq "Failed") -or ($testManifest.State -eq "Cancelled") )
                    {
                        DownloadLogs -testLaunchName $testManifest.TestLaunchName -testName $testManifest.TestName
                    }                    
                    continue
                }            
            }        
        }
        else
        {
            Write-Host "[$(Get-Date -Format 'MM-dd-yy hh:mm')] Checking Test Pass Status ..."
            Write-Host "Test Pass URL $VaaSPortalUri/TestPass/Manage/Summary?solutionName=$VaaSSolutionName&testPassName=$VaaSTestPassName"
            Find-AzureStackVaaSTestLaunch -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName | ft @{Expression={$_.TestName};Label="Test Name"},@{Expression={$_.CreatedTimeUtc};Label="Created"},@{Expression={if($_.StartedTimeUtc -eq "1/1/0001 12:00:00 AM"){""}else{$_.StartedTimeUtc}};Label="Started"},State,AgentName
            $prevTestManifests = $testManifests
        }

        if (!$testPassActive)
        {
            DisplayTitle -title "Test pass completed. Final Result Summary"
            $statusTable = $testManifests | ft @{Expression={$_.TestName};Label="Test Name"},@{Expression={$_.CreatedTimeUtc};Label="Created"},@{Expression={if($_.StartedTimeUtc -eq "1/1/0001 12:00:00 AM"){""}else{$_.StartedTimeUtc}};Label="Started"},State,AgentName
            Write-Host "Test Pass Status:"
            Write-Host "$statusTable"
            foreach($logPath in $launcherData.logPaths)
            {
                Write-Host $logPath
            }

            foreach ($testManifest in $testManifests)
            {
                if( ($testManifest.State -eq "Failed") -or ($testManifest.State -eq "Cancelled") )
                {
                    $launcherData.testResult = 1
                    break
                }
            }
            return
        }

        # If the time interval has been reached, output table, and reset nextInterval
        if($nextInterval -lt (Get-Date))
        {
            Write-Host "[$(Get-Date -Format 'MM-dd-yy hh:mm:ss')] Checking Test Pass Status ..."
            $nextInterval = (Get-Date).AddSeconds($refreshInterval)
        }        

        $prevTestManifests = $testManifests
        Start-Sleep -Seconds $refreshRate | Out-Null
    }
}

function DisplaySessionInfo
{
    Write-Host "Current User: $([Security.Principal.WindowsIdentity]::GetCurrent().Name)"
    $currentPrincipal = New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent())
    if ($currentPrincipal.IsInRole( [Security.Principal.WindowsBuiltInRole]::Administrator ))
    {
        Write-Host "Current session is running as an Administrator ..."
    }
    else
    {
        Write-Warning "Current session is running is not running as an Administrator ..."
    }
}

function DisplayTitle
{
    param(
    [parameter(Mandatory=$true)][string]$title,
    [parameter(Mandatory=$false)][int]$seconds=1
    )
    
    $launcherData.titleCounter++
    Write-Host ""
    Write-Host "----------------------------------------------------" -ForegroundColor Green
    Write-Host " $($launcherData.titleCounter). $title"  -ForegroundColor Green
    Write-Host "----------------------------------------------------" -ForegroundColor Green
    Start-Sleep -s $seconds | Out-Null
}

function Main
{
    Write-Host "Adding AzureStack VaaS Account"
    if($IsProductionRun)
    {
        Write-Host "Production Run ..."
        if($UseVaaSSPNAuth)
        {
            Write-Host "Using SPN Authentication"
            Add-AzureStackVaaSAccount -Credential $VaaSAccountCreds -ServicePrincipal -TenantId $VaaSAccountTenantId -ErrorAction Stop
        }
        else
        {
            Write-Host "Using User Authentication"
            Add-AzureStackVaaSAccount -Credential $VaaSAccountCreds -TenantId $VaaSAccountTenantId -ErrorAction Stop
        }            
    }
    else
    {
        if($UseVaaSSPNAuth)
        {
            Write-Host "Using SPN Authentication"
            Add-AzureStackVaaSAccount -Credential $VaaSAccountCreds -ServicePrincipal -TenantId $VaaSAccountTenantId -TenantServiceBaseUri $VaaSBaseServiceUri -TenantServiceResourceId $VaaSServiceResourceId -ErrorAction Stop
        }
        else
        {
            Write-Host "Using User Authentication"
            Add-AzureStackVaaSAccount -Credential $VaaSAccountCreds -ApplicationId $VaaSApplicationId -ApplicationUri $VaaSApplicationUri -TenantId $VaaSAccountTenantId -TenantServiceBaseUri $VaaSBaseServiceUri -TenantServiceResourceId $VaaSServiceResourceId -ErrorAction Stop
        }        
    }

    $vaasSolution = CreateVaaSSolution
    $vaasTestPass = CreateVaaSTestPass
    $vaasGroupManifest = ConfigureGroupManifestFile

    try
    {
        DisplayTitle -Title "VaaS test pass launch"
        Write-Host "Launching test pass $VaaSTestPassName on Solution $VaaSSolutionName against Agent $VaaSOnPremAgentName"    
        $testLaunchManifests = New-AzureStackVaaSTestGroupLaunch -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName -AgentName $VaaSOnPremAgentName -TestInventoryFilePath $vaasGroupManifest
    }
    catch
    {
        Write-Warning "Unable to launch tests. Attempting retry in 30 seconds ..."
        Start-Sleep -s 30 | Out-Null
        $testLaunchManifests = New-AzureStackVaaSTestGroupLaunch -SolutionName $VaaSSolutionName -TestPassName $VaaSTestPassName -AgentName $VaaSOnPremAgentName -TestInventoryFilePath $vaasGroupManifest -ErrorAction Stop
    }
    
    if(!$testLaunchManifests)
    {
        throw [System.AggregateException] "No test manifest data was returned after launching test group ..."
    }

    Write-Host "You can also monitor your test pass using VaaS Portal at:" -ForegroundColor Green -BackgroundColor Black
    Write-Host "$VaaSPortalUri/TestPass/Manage/Summary?solutionName=$VaaSSolutionName&testPassName=$VaaSTestPassName" -ForegroundColor Green -BackgroundColor Black
    
    if($DoNotWait)
    {
        return    
    }
    
    TestMonitor -testLaunchManifests $testLaunchManifests
    GenerateVaaSReport
}

Set-PSDebug -Strict
$windowTitle = $host.ui.RawUI.WindowTitle
if($LogFilePath -eq "")
{
    $LogFilePath = (Get-Item -Path ".\" -Verbose).FullName + '\Logs'
    Write-Host "Defaulting to VaaS log file path $LogFilePath, as a log file path was not provided ..."    
}

CreateVaaSLogDirectory
$logFileName = "$LogFilePath\LaunchVaaSTests_$(Get-Date -Format yyyyMMdd-HHmmss).log"
Start-Transcript -Path $logFileName -IncludeInvocationHeader
Write-Output "`r`n"
DisplaySessionInfo

try
{
    Write-Output "PsBoundParameters: "
    foreach($keyValue in $PsBoundParameters.GetEnumerator())
    {
        Write-Output "$($keyValue.Key)=$($keyValue.Value)"
    }    
    Write-Output ""

    ValidateAndResolveParameters
    InitializeLaunchData
    InstallVaaSOnPremAgent
    InstallVaaSPreReq

    $host.ui.RawUI.WindowTitle = "VaaS"
    CreateVaaSDirectory
    InstallVaaSPSModules
    Import-Module AzureStackVaaS -Force -ErrorAction Stop
    Write-Host "`r`nAzureStackVaaS module version loaded: $((Get-Module -Name AzureStackVaaS).Version.ToString())`r`n"
    Import-Module 'AzureRM'
    Disable-AzureRmDataCollection -WarningAction Ignore
    StartVaaSOnPremAgent    
    Main
}
catch
{
    $_ | Format-List * -Force | Out-Default;
    $_.InvocationInfo | Format-List * | Out-Default
    $Exception = $_.Exception
    for ($i = 0; $Exception; $i++, ($Exception = $_.InnerException))
    {
      "$i" * 80
      $Exception | Format-List * -Force | Out-Default
    }
    $launcherData.testResult = 1
}
finally
{
    Remove-Module AzureStackVaaS -Force -ErrorAction Ignore
    ShutdownVaaSOnPremAgent
    RemoveVaaSOnPremAgent -agentName $VaaSOnPremAgentName
    Stop-Transcript
    $host.ui.RawUI.WindowTitle = $windowTitle
    exit $launcherData.testResult
}
# SIG # Begin signature block
# MIIdnQYJKoZIhvcNAQcCoIIdjjCCHYoCAQExCzAJBgUrDgMCGgUAMGkGCisGAQQB
# gjcCAQSgWzBZMDQGCisGAQQBgjcCAR4wJgIDAQAABBAfzDtgWUsITrck0sYpfvNR
# AgEAAgEAAgEAAgEAAgEAMCEwCQYFKw4DAhoFAAQUCB2jXSXCMmuOjylW0hj1NcaU
# BzegghhlMIIEwzCCA6ugAwIBAgITMwAAAMhHIp2jDcrAWAAAAAAAyDANBgkqhkiG
# 9w0BAQUFADB3MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4G
# A1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSEw
# HwYDVQQDExhNaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EwHhcNMTYwOTA3MTc1ODU0
# WhcNMTgwOTA3MTc1ODU0WjCBszELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp
# bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw
# b3JhdGlvbjENMAsGA1UECxMETU9QUjEnMCUGA1UECxMebkNpcGhlciBEU0UgRVNO
# Ojk4RkQtQzYxRS1FNjQxMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT
# ZXJ2aWNlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoUNNyknhIcQy
# V4oQO4+cu9wdeLc624e9W0bwCDnHpdxJqtEGkv7f+0kYpyYk8rpfCe+H2aCuA5F0
# XoFWLSkOsajE1n/MRVAH24slLYPyZ/XO7WgMGvbSROL97ewSRZIEkFm2dCB1DRDO
# ef7ZVw6DMhrl5h8s299eDxEyhxrY4i0vQZKKwDD38xlMXdhc2UJGA0GZ16ByJMGQ
# zBqsuRyvxAGrLNS5mjCpogEtJK5CCm7C6O84ZWSVN8Oe+w6/igKbq9vEJ8i8Q4Vo
# hAcQP0VpW+Yg3qmoGMCvb4DVRSQMeJsrezoY7bNJjpicVeo962vQyf09b3STF+cq
# pj6AXzGVVwIDAQABo4IBCTCCAQUwHQYDVR0OBBYEFA/hZf3YjcOWpijw0t+ejT2q
# fV7MMB8GA1UdIwQYMBaAFCM0+NlSRnAK7UD7dvuzK7DDNbMPMFQGA1UdHwRNMEsw
# SaBHoEWGQ2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3Rz
# L01pY3Jvc29mdFRpbWVTdGFtcFBDQS5jcmwwWAYIKwYBBQUHAQEETDBKMEgGCCsG
# AQUFBzAChjxodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY3Jv
# c29mdFRpbWVTdGFtcFBDQS5jcnQwEwYDVR0lBAwwCgYIKwYBBQUHAwgwDQYJKoZI
# hvcNAQEFBQADggEBAJqUDyiyB97jA9U9vp7HOq8LzCIfYVtQfJi5PUzJrpwzv6B7
# aoTC+iCr8QdiMG7Gayd8eWrC0BxmKylTO/lSrPZ0/3EZf4bzVEaUfAtChk4Ojv7i
# KCPrI0RBgZ0+tQPYGTjiqduQo2u4xm0GbN9RKRiNNb1ICadJ1hkf2uzBPj7IVLth
# V5Fqfq9KmtjWDeqey2QBCAG9MxAqMo6Epe0IDbwVUbSG2PzM+rLSJ7s8p+/rxCbP
# GLixWlAtuY2qFn01/2fXtSaxhS4vNzpFhO/z/+m5fHm/j/88yzRvQfWptlQlSRdv
# wO72Vc+Nbvr29nNNw662GxDbHDuGN3S65rjPsAkwggYHMIID76ADAgECAgphFmg0
# AAAAAAAcMA0GCSqGSIb3DQEBBQUAMF8xEzARBgoJkiaJk/IsZAEZFgNjb20xGTAX
# BgoJkiaJk/IsZAEZFgltaWNyb3NvZnQxLTArBgNVBAMTJE1pY3Jvc29mdCBSb290
# IENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0wNzA0MDMxMjUzMDlaFw0yMTA0MDMx
# MzAzMDlaMHcxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYD
# VQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xITAf
# BgNVBAMTGE1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQTCCASIwDQYJKoZIhvcNAQEB
# BQADggEPADCCAQoCggEBAJ+hbLHf20iSKnxrLhnhveLjxZlRI1Ctzt0YTiQP7tGn
# 0UytdDAgEesH1VSVFUmUG0KSrphcMCbaAGvoe73siQcP9w4EmPCJzB/LMySHnfL0
# Zxws/HvniB3q506jocEjU8qN+kXPCdBer9CwQgSi+aZsk2fXKNxGU7CG0OUoRi4n
# rIZPVVIM5AMs+2qQkDBuh/NZMJ36ftaXs+ghl3740hPzCLdTbVK0RZCfSABKR2YR
# JylmqJfk0waBSqL5hKcRRxQJgp+E7VV4/gGaHVAIhQAQMEbtt94jRrvELVSfrx54
# QTF3zJvfO4OToWECtR0Nsfz3m7IBziJLVP/5BcPCIAsCAwEAAaOCAaswggGnMA8G
# A1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCM0+NlSRnAK7UD7dvuzK7DDNbMPMAsG
# A1UdDwQEAwIBhjAQBgkrBgEEAYI3FQEEAwIBADCBmAYDVR0jBIGQMIGNgBQOrIJg
# QFYnl+UlE/wq4QpTlVnkpKFjpGEwXzETMBEGCgmSJomT8ixkARkWA2NvbTEZMBcG
# CgmSJomT8ixkARkWCW1pY3Jvc29mdDEtMCsGA1UEAxMkTWljcm9zb2Z0IFJvb3Qg
# Q2VydGlmaWNhdGUgQXV0aG9yaXR5ghB5rRahSqClrUxzWPQHEy5lMFAGA1UdHwRJ
# MEcwRaBDoEGGP2h0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1
# Y3RzL21pY3Jvc29mdHJvb3RjZXJ0LmNybDBUBggrBgEFBQcBAQRIMEYwRAYIKwYB
# BQUHMAKGOGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljcm9z
# b2Z0Um9vdENlcnQuY3J0MBMGA1UdJQQMMAoGCCsGAQUFBwMIMA0GCSqGSIb3DQEB
# BQUAA4ICAQAQl4rDXANENt3ptK132855UU0BsS50cVttDBOrzr57j7gu1BKijG1i
# uFcCy04gE1CZ3XpA4le7r1iaHOEdAYasu3jyi9DsOwHu4r6PCgXIjUji8FMV3U+r
# kuTnjWrVgMHmlPIGL4UD6ZEqJCJw+/b85HiZLg33B+JwvBhOnY5rCnKVuKE5nGct
# xVEO6mJcPxaYiyA/4gcaMvnMMUp2MT0rcgvI6nA9/4UKE9/CCmGO8Ne4F+tOi3/F
# NSteo7/rvH0LQnvUU3Ih7jDKu3hlXFsBFwoUDtLaFJj1PLlmWLMtL+f5hYbMUVbo
# nXCUbKw5TNT2eb+qGHpiKe+imyk0BncaYsk9Hm0fgvALxyy7z0Oz5fnsfbXjpKh0
# NbhOxXEjEiZ2CzxSjHFaRkMUvLOzsE1nyJ9C/4B5IYCeFTBm6EISXhrIniIh0EPp
# K+m79EjMLNTYMoBMJipIJF9a6lbvpt6Znco6b72BJ3QGEe52Ib+bgsEnVLaxaj2J
# oXZhtG6hE6a/qkfwEm/9ijJssv7fUciMI8lmvZ0dhxJkAj0tr1mPuOQh5bWwymO0
# eFQF1EEuUKyUsKV4q7OglnUa2ZKHE3UiLzKoCG6gW4wlv6DvhMoh1useT8ma7kng
# 9wFlb4kLfchpyOZu6qeXzjEp/w7FW1zYTRuh2Povnj8uVRZryROj/TCCBhEwggP5
# oAMCAQICEzMAAACOh5GkVxpfyj4AAAAAAI4wDQYJKoZIhvcNAQELBQAwfjELMAkG
# A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx
# HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z
# b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMTAeFw0xNjExMTcyMjA5MjFaFw0xODAy
# MTcyMjA5MjFaMIGDMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQ
# MA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9u
# MQ0wCwYDVQQLEwRNT1BSMR4wHAYDVQQDExVNaWNyb3NvZnQgQ29ycG9yYXRpb24w
# ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQh9RCK36d2cZ61KLD4xWS
# 0lOdlRfJUjb6VL+rEK/pyefMJlPDwnO/bdYA5QDc6WpnNDD2Fhe0AaWVfIu5pCzm
# izt59iMMeY/zUt9AARzCxgOd61nPc+nYcTmb8M4lWS3SyVsK737WMg5ddBIE7J4E
# U6ZrAmf4TVmLd+ArIeDvwKRFEs8DewPGOcPUItxVXHdC/5yy5VVnaLotdmp/ZlNH
# 1UcKzDjejXuXGX2C0Cb4pY7lofBeZBDk+esnxvLgCNAN8mfA2PIv+4naFfmuDz4A
# lwfRCz5w1HercnhBmAe4F8yisV/svfNQZ6PXlPDSi1WPU6aVk+ayZs/JN2jkY8fP
# AgMBAAGjggGAMIIBfDAfBgNVHSUEGDAWBgorBgEEAYI3TAgBBggrBgEFBQcDAzAd
# BgNVHQ4EFgQUq8jW7bIV0qqO8cztbDj3RUrQirswUgYDVR0RBEswSaRHMEUxDTAL
# BgNVBAsTBE1PUFIxNDAyBgNVBAUTKzIzMDAxMitiMDUwYzZlNy03NjQxLTQ0MWYt
# YmM0YS00MzQ4MWU0MTVkMDgwHwYDVR0jBBgwFoAUSG5k5VAF04KqFzc3IrVtqMp1
# ApUwVAYDVR0fBE0wSzBJoEegRYZDaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3Br
# aW9wcy9jcmwvTWljQ29kU2lnUENBMjAxMV8yMDExLTA3LTA4LmNybDBhBggrBgEF
# BQcBAQRVMFMwUQYIKwYBBQUHMAKGRWh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9w
# a2lvcHMvY2VydHMvTWljQ29kU2lnUENBMjAxMV8yMDExLTA3LTA4LmNydDAMBgNV
# HRMBAf8EAjAAMA0GCSqGSIb3DQEBCwUAA4ICAQBEiQKsaVPzxLa71IxgU+fKbKhJ
# aWa+pZpBmTrYndJXAlFq+r+bltumJn0JVujc7SV1eqVHUqgeSxZT8+4PmsMElSnB
# goSkVjH8oIqRlbW/Ws6pAR9kRqHmyvHXdHu/kghRXnwzAl5RO5vl2C5fAkwJnBpD
# 2nHt5Nnnotp0LBet5Qy1GPVUCdS+HHPNIHuk+sjb2Ns6rvqQxaO9lWWuRi1XKVjW
# kvBs2mPxjzOifjh2Xt3zNe2smjtigdBOGXxIfLALjzjMLbzVOWWplcED4pLJuavS
# Vwqq3FILLlYno+KYl1eOvKlZbiSSjoLiCXOC2TWDzJ9/0QSOiLjimoNYsNSa5jH6
# lEeOfabiTnnz2NNqMxZQcPFCu5gJ6f/MlVVbCL+SUqgIxPHo8f9A1/maNp39upCF
# 0lU+UK1GH+8lDLieOkgEY+94mKJdAw0C2Nwgq+ZWtd7vFmbD11WCHk+CeMmeVBoQ
# YLcXq0ATka6wGcGaM53uMnLNZcxPRpgtD1FgHnz7/tvoB3kH96EzOP4JmtuPe7Y6
# vYWGuMy8fQEwt3sdqV0bvcxNF/duRzPVQN9qyi5RuLW5z8ME0zvl4+kQjOunut6k
# LjNqKS8USuoewSI4NQWF78IEAA1rwdiWFEgVr35SsLhgxFK1SoK3hSoASSomgyda
# Qd691WZJvAuceHAJvDCCB3owggVioAMCAQICCmEOkNIAAAAAAAMwDQYJKoZIhvcN
# AQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYD
# VQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xMjAw
# BgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAyMDEx
# MB4XDTExMDcwODIwNTkwOVoXDTI2MDcwODIxMDkwOVowfjELMAkGA1UEBhMCVVMx
# EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT
# FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9zb2Z0IENvZGUg
# U2lnbmluZyBQQ0EgMjAxMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB
# AKvw+nIQHC6t2G6qghBNNLrytlghn0IbKmvpWlCquAY4GgRJun/DDB7dN2vGEtgL
# 8DjCmQawyDnVARQxQtOJDXlkh36UYCRsr55JnOloXtLfm1OyCizDr9mpK656Ca/X
# llnKYBoF6WZ26DJSJhIv56sIUM+zRLdd2MQuA3WraPPLbfM6XKEW9Ea64DhkrG5k
# NXimoGMPLdNAk/jj3gcN1Vx5pUkp5w2+oBN3vpQ97/vjK1oQH01WKKJ6cuASOrdJ
# Xtjt7UORg9l7snuGG9k+sYxd6IlPhBryoS9Z5JA7La4zWMW3Pv4y07MDPbGyr5I4
# ftKdgCz1TlaRITUlwzluZH9TupwPrRkjhMv0ugOGjfdf8NBSv4yUh7zAIXQlXxgo
# tswnKDglmDlKNs98sZKuHCOnqWbsYR9q4ShJnV+I4iVd0yFLPlLEtVc/JAPw0Xpb
# L9Uj43BdD1FGd7P4AOG8rAKCX9vAFbO9G9RVS+c5oQ/pI0m8GLhEfEXkwcNyeuBy
# 5yTfv0aZxe/CHFfbg43sTUkwp6uO3+xbn6/83bBm4sGXgXvt1u1L50kppxMopqd9
# Z4DmimJ4X7IvhNdXnFy/dygo8e1twyiPLI9AN0/B4YVEicQJTMXUpUMvdJX3bvh4
# IFgsE11glZo+TzOE2rCIF96eTvSWsLxGoGyY0uDWiIwLAgMBAAGjggHtMIIB6TAQ
# BgkrBgEEAYI3FQEEAwIBADAdBgNVHQ4EFgQUSG5k5VAF04KqFzc3IrVtqMp1ApUw
# GQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB
# /wQFMAMBAf8wHwYDVR0jBBgwFoAUci06AjGQQ7kUBU7h6qfHMdEjiTQwWgYDVR0f
# BFMwUTBPoE2gS4ZJaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJv
# ZHVjdHMvTWljUm9vQ2VyQXV0MjAxMV8yMDExXzAzXzIyLmNybDBeBggrBgEFBQcB
# AQRSMFAwTgYIKwYBBQUHMAKGQmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kv
# Y2VydHMvTWljUm9vQ2VyQXV0MjAxMV8yMDExXzAzXzIyLmNydDCBnwYDVR0gBIGX
# MIGUMIGRBgkrBgEEAYI3LgMwgYMwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWlj
# cm9zb2Z0LmNvbS9wa2lvcHMvZG9jcy9wcmltYXJ5Y3BzLmh0bTBABggrBgEFBQcC
# AjA0HjIgHQBMAGUAZwBhAGwAXwBwAG8AbABpAGMAeQBfAHMAdABhAHQAZQBtAGUA
# bgB0AC4gHTANBgkqhkiG9w0BAQsFAAOCAgEAZ/KGpZjgVHkaLtPYdGcimwuWEeFj
# kplCln3SeQyQwWVfLiw++MNy0W2D/r4/6ArKO79HqaPzadtjvyI1pZddZYSQfYtG
# UFXYDJJ80hpLHPM8QotS0LD9a+M+By4pm+Y9G6XUtR13lDni6WTJRD14eiPzE32m
# kHSDjfTLJgJGKsKKELukqQUMm+1o+mgulaAqPyprWEljHwlpblqYluSD9MCP80Yr
# 3vw70L01724lruWvJ+3Q3fMOr5kol5hNDj0L8giJ1h/DMhji8MUtzluetEk5CsYK
# wsatruWy2dsViFFFWDgycScaf7H0J/jeLDogaZiyWYlobm+nt3TDQAUGpgEqKD6C
# PxNNZgvAs0314Y9/HG8VfUWnduVAKmWjw11SYobDHWM2l4bf2vP48hahmifhzaWX
# 0O5dY0HjWwechz4GdwbRBrF1HxS+YWG18NzGGwS+30HHDiju3mUv7Jf2oVyW2ADW
# oUa9WfOXpQlLSBCZgB/QACnFsZulP0V3HjXG0qKin3p6IvpIlR+r+0cjgPWe+L9r
# t0uX4ut1eBrs6jeZeRhL/9azI2h15q/6/IvrC4DqaTuv/DDtBEyO3991bWORPdGd
# Vk5Pv4BXIqF4ETIheu9BCrE/+6jMpF3BoYibV3FWTkhFwELJm3ZbCoBIa/15n8G9
# bW1qyVJzEw16UM0xggSiMIIEngIBATCBlTB+MQswCQYDVQQGEwJVUzETMBEGA1UE
# CBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9z
# b2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5n
# IFBDQSAyMDExAhMzAAAAjoeRpFcaX8o+AAAAAACOMAkGBSsOAwIaBQCggbYwGQYJ
# KoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEOMAwGCisGAQQB
# gjcCARUwIwYJKoZIhvcNAQkEMRYEFAoMc4EcYhm7gYE9mVjBQVElT6ufMFYGCisG
# AQQBgjcCAQwxSDBGoCiAJgBMAGEAdQBuAGMAaABWAGEAYQBTAFQAZQBzAHQAcwAu
# AHAAcwAxoRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTANBgkqhkiG9w0BAQEF
# AASCAQC0krOBtDg631Eh4U9g7Jzxjt88IVRVPnaCzn+JhNX4u6SkBZTX0aM/tIIF
# TBuks41wkswX2N54512VfXP9CAS/g4WXIiQfvBbK+3CQY/bkvFGXCT0gTXqHDrRI
# wz8vouGO2Mr1+kPrlW/ZCUexGabtwVBBZOAbkzPQhnm9dUVQdXPxLx+SSd5DFBAt
# gBcIi/V4f/hYwbotFw533QgDnXgBz8xTS6Ht9rOEKg8Kcmwn+vHa+Y7VOapLUadI
# +uzdlm85mpa2mUzSADMFn3hzhtx23GfQg/Q4hDpF+LLyKWcpy9sQdsuAEJswuf0b
# 38BKLfosAOXNkUk3+X3E4odcyPlGoYICKDCCAiQGCSqGSIb3DQEJBjGCAhUwggIR
# AgEBMIGOMHcxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYD
# VQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xITAf
# BgNVBAMTGE1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQQITMwAAAMhHIp2jDcrAWAAA
# AAAAyDAJBgUrDgMCGgUAoF0wGAYJKoZIhvcNAQkDMQsGCSqGSIb3DQEHATAcBgkq
# hkiG9w0BCQUxDxcNMTcwMzA3MjMyNDU4WjAjBgkqhkiG9w0BCQQxFgQUvUYVaeWa
# FNKec8ZJjXdG5P7g+RkwDQYJKoZIhvcNAQEFBQAEggEAhRoLnxfy69MCKjBrnNx2
# pjlpLk/TtkkBIjfU3Z5UvynsgNq7JyjuedQ13GUdEtDlIrk3M79j3fBQC9U0x9gl
# Pkqajug7tOA4ZTW+dPSe6hiC3rfedaFsMOsMA6hrI3zcfhgpk9nUme6H6EcUc56+
# w8mgjOpjpeI2Q1IyunmSMxFvpd3yd9zbDhF8JAn7fvZU+alXKoLsK8UoeXNBKVx1
# PMh4uhmFi29ek6Uk6to+eOY/pYssHvORw3qysTz5bDxmbmt+FhkSyq4rnrDZ68/L
# tgDnZeBcH6cZ5gVpGnS0eCjUymDgzd0LR85C8Q3Ja3zmaVRjZcbik8bQD/3F8Q3C
# pg==
# SIG # End signature block