AzStackHci.EnvironmentChecker

1.2100.2410.204

Obs/bin/ObsDep/content/Powershell/Roles/Common/RoleHelpers.psm1

<###################################################
 # #
 # Copyright (c) Microsoft. All rights reserved. #
 # #
 ##################################################>

Import-Module $PSScriptRoot\JustEnoughAdministrationHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null
Import-Module NetworkControllerRESTWrappers -DisableNameChecking -Verbose:$false | Out-Null
Import-Module $PSScriptRoot\NetworkControllerWorkloadHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null
Import-Module $PSScriptRoot\..\..\Common\Helpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null
Import-Module $PSScriptRoot\..\..\Common\NetworkHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null
Import-Module $PSScriptRoot\..\..\Common\StorageHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null
Import-Module $PSScriptRoot\..\..\Common\ClusterHelpers.psm1 -DisableNameChecking -Verbose:$false | Out-Null

Import-Module PSDiagnostics -DisableNameChecking -Verbose:$false | Out-Null

Import-LocalizedData LocalizedStrings -FileName Roles.Strings.psd1 -ErrorAction SilentlyContinue
Import-LocalizedData LocalizedNetworkData -FileName Network.Strings.psd1 -ErrorAction SilentlyContinue

$GUEST_VM_REBOOT_TIMEOUT_IN_MINUTES = 10

#Checks if the value is null or empty and throws the exception
function Throw-IfNullOrEmpty
{
    Param
    (
        [object]
        $DataToCheck,

        [string]
        $ErrorMessage
    )
     if ($DataToCheck)
     {
        if (([string]::IsNullOrEmpty($DataToCheck)) -or ($DataToCheck.Length -eq 0))
        {
            throw $ErrorMessage
        }
    }
    else
    {
        throw $ErrorMessage
    }
}

#Gets the CA certificate password
function Get-CACertPassword
{
    Param
  (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
    Throw-IfNullOrEmpty -DataToCheck $cloudRole -ErrorMessage "Unable to get cloudRole information from ECE"

    $securityInfo = $cloudRole.PublicInfo.SecurityInfo
    Throw-IfNullOrEmpty -DataToCheck $securityInfo -ErrorMessage "Unable to get securityInfo information from ECE"

    $caCertUser = $securityInfo.CACertUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.CACertAccountID
    Throw-IfNullOrEmpty -DataToCheck $caCertUser -ErrorMessage "Unable to get caCertUser information from ECE"

    $secureCertPwd = $Parameters.GetCredential($caCertUser.Credential).Password
    Throw-IfNullOrEmpty -DataToCheck $secureCertPwd -ErrorMessage "Unable to get secureCertPwd information from ECE"

  return $secureCertPwd
}

#Gets the External Domain FQDN
function Get-ExternalDomainFQDN
{
    Param
  (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

  $domainRole = $Parameters.Roles["Domain"].PublicConfiguration
  Throw-IfNullOrEmpty -DataToCheck $domainRole -ErrorMessage "Unable to get domainRole information from ECE"

  $externalDomainFqdn = $domainRole.PublicInfo.DomainConfiguration.EXTERNALFQDN
  Throw-IfNullOrEmpty -DataToCheck $externalDomainFqdn -ErrorMessage "Unable to get External DomainFQDN information from ECE"

  return $externalDomainFqdn

  }

#Gets the Internal Domain FQDN
function Get-InternalDomainFQDN
{
    Param
  (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

  $domainRole = $Parameters.Roles["Domain"].PublicConfiguration
  Throw-IfNullOrEmpty -DataToCheck $domainRole -ErrorMessage "Unable to get domainRole information from ECE"

    $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN
  Throw-IfNullOrEmpty -DataToCheck $domainFqdn -ErrorMessage "Unable to get DomainFQDN information from ECE"

  return $domainFqdn

}

function Test-PSSession {
    Param(
        [Parameter(Mandatory=$true)]
        $RemoteSession,

        [Parameter(Mandatory=$true)]
        $SessionCredential = $true,

        [Parameter(Mandatory=$true)]
        $SessionComputer = $true
    )

    if (($RemoteSession -eq $null) -or ($RemoteSession.State -ne "Opened"))
    {
        Trace-Execution "Session not ready, state: $($RemoteSession.State)"
        Trace-Execution "Removing the session"
        $RemoteSession | Remove-PSSession

        Trace-Execution "Regenerating the session"
        $RemoteSession = New-PSSession -ComputerName $SessionComputer -Credential $SessionCredential

        Initialize-ECESession $RemoteSession
        Trace-Execution "State after regenerate = $($RemoteSession.State)"
    } else {
        Trace-Execution "Session Opened"
    }

    return $RemoteSession
}

function Test-SFRingHealth {
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    Trace-Execution "Starting HealthCheck for $($Parameters.Configuration.Role.ID)."

    $clusterConnInfo = Get-SFClusterConnectionInfo $Parameters
    $AzureStackNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.Fabric"
    $verifyClusterHealth = Join-Path $AzureStackNugetPath "content\AzureStackInstaller\Utils\WinFabricCluster\VerifyClusterHealth.psm1"
    import-module $verifyClusterHealth

    try {
        Trace-Execution "Checking health of Service Fabric cluster..."
        VerifyClusterConnection -ServiceFabricEndpoint $clusterConnInfo.sfClusterEndpoints -ClusterSpn $clusterConnInfo.clusterSpn -CheckAggregateHealthState -RetryTimes 10 -RetryIntervalSec 15 -ConnectionTimeoutSec 30 -Verbose
    }
    catch {
        Trace-Execution "Check health of Service Fabric cluster failed."
        Trace-Warning $_
        Trace-Execution "Attempting to repair Service Fabric cluster..."
        Connect-AzureStackServiceFabricCluster -ConnectionEndpoint $clusterConnInfo.sfClusterEndpoints -ClusterSpn $clusterSpn
        $repairServiceFabricClusterScript = Join-Path $AzureStackNugetPath "content\AzureStackInstaller\Utils\WinFabricCluster\RepairServiceFabricCluster.psm1"
        Import-Module $repairServiceFabricClusterScript -DisableNameChecking -Verbose:$false | Out-Null
        Repair-AzureStackServiceFabricCluster -Verbose

        Trace-Execution "Rechecking health of Service Fabric cluster..."
        VerifyClusterConnection -ServiceFabricEndpoint $clusterConnInfo.sfClusterEndpoints -ClusterSpn $clusterConnInfo.clusterSpn -CheckAggregateHealthState -RetryTimes 10 -RetryIntervalSec 15 -ConnectionTimeoutSec 30 -Verbose
    }
}

function Test-PrecheckClusterNodeHealth {
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $node = Get-ExecutionContextNodeName $Parameters

    Trace-Execution "Starting Node Health Check for $node on $($Parameters.Configuration.Role.ID)."

    $clusterConnInfo = Get-SFClusterConnectionInfo $Parameters
    $AzureStackNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.Fabric"
    $verifyClusterHealth = Join-Path $AzureStackNugetPath "content\AzureStackInstaller\Utils\WinFabricCluster\VerifyClusterHealth.psm1"
    Import-Module $verifyClusterHealth

    Trace-Execution "Checking Node health from Service fabric..."
    PrecheckClusterNodeHealth -ServiceFabricEndpoint $clusterConnInfo.sfClusterEndpoints -ClusterSpn $clusterConnInfo.clusterSpn -NodeExemptIP $node -ConnectionTimeoutSec 30 -IncludeDisabled $true -Verbose
}

function Test-PrecheckNCClusterNodeHealth {
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $roleDefinition = $Parameters.Configuration.Role
    [string] $TimeString = Get-Date -Format "yyyyMMdd-HHmmss"
    $RemoteLOGFILE = "$env:systemdrive\MASLogs\$($RoleDefinition.Id)_$($MyInvocation.MyCommand.Name)_$TimeString.log"

    $securityInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo

        #Setup Account Info
    $setupAccountUser = $securityInfo.DomainUsers.User | ? Role -EQ $roleDefinition.PrivateInfo.Accounts.RunAsAccountID
    $fabricAdminCredential = $Parameters.GetCredential($setupAccountUser.Credential)
    if (-not $("$($fabricAdminCredential.UserName)" -like "*\*")) {
        $fabricAdminCredential = New-Credential -UserName "$domainName\$($fabricAdminCredential.UserName)" -Password $fabricAdminCredential.GetNetworkCredential().Password
    }
    Trace-Execution "Using $($fabricAdminCredential.UserName) to communicate with NC"

    $node = Get-ExecutionContextNodeName $Parameters

    # Get VM names
    [string[]] $vms = $roleDefinition.Nodes.Node | ? { $_.Name -ne $node } | % Name

    Trace-Execution "Starting Node Health Check with context $node on $($roleDefinition.ID)."

    $clusterConnInfo = Get-SFClusterConnectionInfo $Parameters

    $scriptBlock = {
        param($clusterConnInfo, $node)

        Start-Transcript -Append -Path $using:RemoteLOGFILE

        try{
            $AzureStackNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.Fabric"
            $verifyClusterHealth = Join-Path $AzureStackNugetPath "content\AzureStackInstaller\Utils\WinFabricCluster\NC\VerifyClusterHealth.psm1"
            Import-Module $verifyClusterHealth

            Trace-Execution "Checking Node health from Service fabric..."
            PrecheckClusterNodeHealth -ClusterSpn $clusterConnInfo.clusterSpn -NodeExemptIP $node -ConnectionTimeoutSec 30 -IncludeDisabled $true -Verbose
        }finally{
            Stop-Transcript -ErrorAction Ignore
        }
    }

    [System.Management.Automation.Runspaces.PSSession[]] $remoteSession = $null
    try {
        $remoteSession = New-PSSession -ComputerName $vms[0] -Credential $fabricAdminCredential -Authentication Credssp

        $null = Invoke-Command -Session $remoteSession -ScriptBlock $ScriptBlock -ArgumentList $clusterConnInfo, $node
    }
    finally {
        $remoteSession | Remove-Pssession -ErrorAction Ignore | Out-Null
    }

}

function Get-SFClusterConnectionInfo {
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $domainRole = $Parameters.Roles["Domain"].PublicConfiguration
    $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN

    $clusterSpn = $Parameters.Configuration.Role.PublicInfo.ClusterSpn.Name
    [string[]] $sfClusterEndpoints = $Parameters.Configuration.Role.Nodes.Node.Name | ForEach-Object { $_ + "." + $domainFqdn + ":19000" }

    return @{ClusterSPN=$clusterSpn
                    SFClusterEndpoints=$sfClusterEndpoints}
}

# This function creates the role VM clusters as described in the manifest.
function Add-GuestCluster {

    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $true)]
        [String]
        $RunAsUserID
    )

    $ErrorActionPreference = 'Stop'

    $vmRole = $Parameters.Configuration.Role

    # Account info
    $secInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo

    #Setup Account Info
    $setupAccountUser = $secInfo.DomainUsers.User | ? Role -EQ $RunAsUserID
    $setupAccountCredential = $Parameters.GetCredential($setupAccountUser.Credential)

    $domainFqdn = $Parameters.Roles["Domain"].PublicConfiguration.PublicInfo.DomainConfiguration.FQDN
    $null = ipconfig /flushdns

    [array] $vms = $vmRole.Nodes.Node | % Name | Foreach-Object {"$_`.$domainFqdn"}
    $clusterName = $vmRole.PublicInfo.Cluster.Name
    $clusterIPAddress = $vmRole.PublicInfo.Cluster.IPv4Address

    [array] $VMsToAdd = @()
    $Cluster = $null
    $createNewCluster = $true

    try
    {
        $Cluster = Get-Cluster -Name $clusterName
    }
    catch
    {
    }

    if($Cluster -ne $null)
    {
        Trace-Warning "The cluster by the name '$clusterName' already exists."
        $createNewCluster = $false

        [array] $preClusteredVMs = Get-Cluster -Name $clusterName | Get-ClusterNode | % Name | Foreach-Object {"$_`.$domainFqdn"}
        [array] $newVMsToAdd = @()

        foreach($vm in $vms)
        {
            if($preClusteredVMs -contains $vm)
            {
                $result = $true
            }
            else
            {
                $newVMsToAdd += $vm
            }
        }

        if($newVMsToAdd.Count -eq 0)
        {
            # In the event all nodes are part of the cluster, we should still run cluster validation to rule out failures from previous attempts to add-clusternode
            Trace-Execution "Skipping: Nodes '$($preClusteredVMs -join ',')' are already part of Cluster '$clusterName'."
            Trace-Execution "Testing cluster '$clusterName'"
            $null = Test-Cluster -Cluster $clusterName -Verbose -ErrorAction Stop
            return
        }
        else
        {
          $VMsToAdd = $newVMsToAdd
        }
    }
    else
    {
        $VMsToAdd = $vms
    }

    Trace-Execution -Message "Prepare nodes for clustering: '$($VMsToAdd -join ',')'"
    $vmsPendingFeatureInstall = $VMsToAdd | % { if (-not (Get-WindowsFeature -Name Failover-Clustering -ComputerName "$_" -Credential $setupAccountCredential).Installed) { $_ } }

    if ($vmsPendingFeatureInstall) {
        Trace-Execution "Installing Failover-Clustering feature on the following VMs: '$($vmsPendingFeatureInstall -join ',')'."
        $null = $vmsPendingFeatureInstall |
            % { Invoke-Command -ComputerName "$_" -Credential $setupAccountCredential -ErrorAction Stop -ScriptBlock {
                    Add-WindowsFeature -Name Failover-Clustering -IncludeAllSubFeature -IncludeManagementTools
                }
              }

        Trace-Execution "Rebooting the following machines: '$($vmsPendingFeatureInstall -join ',')'."
        Restart-Machine -ComputerName $vmsPendingFeatureInstall -Credential $setupAccountCredential
        Trace-Execution "Waiting for 1 minute to allow for post-installation configuration."
        Start-Sleep -Seconds 60
        Trace-Execution "Waiting for clustering features to become available on following VMs after reboot: '$($vmsPendingFeatureInstall -join ',')'."
        foreach ($vm in $vmsPendingFeatureInstall) {
            Trace-Execution "Waiting for clustering features on '$vm'."
            $waitScript = {
                $null = ipconfig /flushdns
                Invoke-Command -ComputerName $vm -Credential $setupAccountCredential { Get-Command Get-Cluster -ErrorAction Stop }
            }
            if (Wait-Result -ValidationScript $waitScript -TimeOut ($GUEST_VM_REBOOT_TIMEOUT_IN_MINUTES * 60) -Interval 10) {
                Trace-Execution "Clustering feature is now available on '$vm'."
            }
            else {
                Trace-Error "The VM '$vm' didn't reboot in $GUEST_VM_REBOOT_TIMEOUT_IN_MINUTES minutes or the clustering features could not be installed."
            }
        }
    }

    foreach($vm in $VMsToAdd){
        Trace-Execution "Enabling client/diagnostic event log on $vm"

        Invoke-Command -ComputerName $vm -Credential $setupAccountCredential -Scriptblock {
            $logName = 'Microsoft-Windows-FailoverClustering-Client/Diagnostic'
            $log = New-Object System.Diagnostics.Eventing.Reader.EventLogConfiguration $logName
            if($log.IsEnabled -ne $true){
                $log.IsEnabled = $true
                $log.SaveChanges()
            }
        }
    }

    try
    {
        if($createNewCluster)
        {
            $domainCredential = Get-DomainCredential -Parameters $Parameters

            Remove-ComputerAndDnsRecord -Name $clusterName -DomainFQDN $domainFqdn -Credential $domainCredential

            Trace-Execution "Creating the cluster '$clusterName'."
            Invoke-Command -ComputerName $env:COMPUTERNAME -Credential $domainCredential -Authentication Credssp -Scriptblock {
                    $null = New-Cluster -Name $using:clusterName -Node $using:VMsToAdd -StaticAddress $using:clusterIPAddress -NoStorage -ErrorAction Stop
                } -ErrorAction Stop
        }
        else
        {
            # We need new cluster reports if this fails
            Trace-Execution -Message "Adding the New VMs: '$($VMsToAdd -join ',')' to Pre-existing Cluster $clusterName"
            try
            {
                Add-ClusterNode -Name $VMsToAdd -NoStorage -Cluster $clusterName -Verbose -ErrorAction Stop
            }
            finally
            {
                Trace-Execution "Testing cluster '$clusterName'"
                $null = Test-Cluster -Cluster $clusterName -Verbose -ErrorAction Stop
            }
        }

        # We should validate that the cluster is in an expected state
        Trace-Execution "Wait until new cluster nodes can be enumerated on the name '$clusterName'."
        $timeOut = 30 # 30 minutes
        $waitResult = Wait-Result {
            $null = ipconfig /flushdns
            $cluster = Get-Cluster -Domain $domainFqdn | ? Name -eq $clusterName
            $clusterNodes = $cluster | Get-ClusterNode -WarningAction SilentlyContinue | ? { % {[system.net.dns]::GetHostByName($_).HostName -in $VMsToAdd } }
            if(-not $clusterNodes){
                Trace-Warning "No VMsToAdd were discovered. Waiting for: '$VMsToAdd'"
                return $false
            }

            $offlineNodes = $clusterNodes | ? { $_.State -ine "Up" } # We round-robin VM update, so by this point no nodes should be offline
            $clusterNodesFQDN = $clusterNodes | % {[system.net.dns]::GetHostByName($_).HostName}
            $allEnumerated = Compare-Object -ReferenceObject $VMsToAdd -DifferenceObject $clusterNodesFQDN
            if($offlineNodes)
            {
                Trace-Execution "Not all nodes are online: '$($offlineNodes.Name)'"
                return $false
            }
            elseif($allEnumerated) # clusterNodes should be the complete list of VMsToAdd
            {
                Trace-Execution "Not all nodes were enumerated"
                Trace-Execution "Expected nodes: '$VMsToAdd'"
                Trace-Execution "Found nodes: '$($clusterNodes.Name)'"
                return $false
            }
            else
            {
                return $true
            }
        } -TimeOut ($timeOut*60) -Interval 5
        if (-not $waitResult)
        {
            $cluster = Get-Cluster -Domain $domainFqdn | ? Name -eq $clusterName
            $clusterNodes = $cluster | Get-ClusterNode -WarningAction SilentlyContinue
            $clusterResources = Invoke-ScriptBlockWithRetries -RetryTimes 5 -RetrySleepTimeInSeconds 1 -ScriptBlock {$cluster | Get-ClusterResource}
            $clusterNodeStates = ""
            $clusterResourceStates = ""
            foreach($node in $clusterNodes){
                $clusterNodeStates += "'$($node.Name)' is '$($node.State)'; "
            }
            foreach ($resource in $clusterResources){
                $clusterResourceStates += "'$($resource.Name)' is '$($resource.State)'; "
            }
            Trace-Execution "VMs to add to cluster '$clusterName' included: '$VMsToAdd'"
            Trace-Execution "Nodes of the cluster '$clusterName' are in state: '$clusterNodeStates'"
            Trace-Execution "Cluster resources are in state: '$clusterResourceStates'"
            Trace-Error "Nodes of the cluster '$clusterName' did not come online in '$timeOut' minutes."
        }
        else {
            Trace-Execution "Cluster '$clusterName' is now online."
        }

    }
    catch
    {
        Trace-Error $_
    }
    finally
    {
        foreach($vm in $VMsToAdd)
        {
            Trace-Execution "Collecting client/diagnostic event log on $vm"

            # Have to disable and collect winevent, and eventlog does not have the client/diagnostic information
            # To avoid breaking engine execution, we'll print any failures to the log directory. Failure to collect logs should not break procedures.
            try
            {
                Invoke-Command -ComputerName $vm -Credential $setupAccountCredential -Scriptblock {
                        $writePath = "$($env:systemDrive)\AzSLogs"
                        if(!(Test-Path $writePath))
                        {
                            mkdir $writePath
                        }

                        $logName = 'Microsoft-Windows-FailoverClustering-Client/Diagnostic'
                        $log = New-Object System.Diagnostics.Eventing.Reader.EventLogConfiguration $logName
                        if($log.IsEnabled -ne $false){
                            $log.IsEnabled = $false
                            $log.SaveChanges()
                        }

                        Get-ClusterLog -Cluster $using:clusterName -Node $using:vm -Destination "$writePath\$(Get-Date -f yyyyMMdd-hhmmss)"
                        Get-WinEvent -LogName $logName -Oldest | Format-Table TimeCreated, Id, LevelDisplayName, Message -wrap -autosize | Out-File "$writePath\Cluster_Client_Diagnostics_$(Get-Date -f yyyyMMdd-hhmmss).txt"
                     }
            }
            catch
            {
                Trace-Warning "Issue with collecting dlient/diagnostic event log: $_"
            }
        }
    }
}

# This function gets the machine name of the primary server handling external DNS zones.
function Get-ExternalDnsMachineName {

    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,
        [Parameter(Mandatory = $false)]
        [bool]
        $ErrorOnFail = $true
    )

    $ErrorActionPreference = 'Stop'
    $waspRole = $Parameters.Roles["WASPUBLIC"].PublicConfiguration
    $externalDnsRole = $Parameters.Roles["ExternalDNS"].PublicConfiguration

    $DnsServerNamesList = $waspRole.Nodes.Node | Foreach-Object { $_.Name }
    $ExternalZoneFqdn = $externalDnsRole.PublicInfo.DnsConfiguration.ExternalZoneFqdn

    $domainAdminCredential = Get-DomainCredential -Parameters $Parameters
    $domainAdminCimSession = New-CimSession -Credential $domainAdminCredential

    Trace-Execution "Finding primary DNS server for external zone '$ExternalZoneFqdn' in list '$DnsServerNamesList'."

    $PrimaryDnsServerNames = @()
    foreach($DnsServer in $DnsServerNamesList)
    {
        try
        {
            $zone = Get-DnsServerZone -Name $ExternalZoneFqdn -ComputerName $DnsServer -CimSession $domainAdminCimSession
            if ($zone.ZoneType -eq "Primary")
            {
                Trace-Execution "Found primary DNS server '$DnsServer'."
                $PrimaryDnsServerNames += $DnsServer
            }
        }
        catch
        {
            Write-Verbose "Get-ExternalDnsMachineName failed to get zone from $DnsServer. Suppressing Exception $_"
        }
    }

    if ($PrimaryDnsServerNames.Count -eq 0)
    {
        if ($ErrorOnFail)
        {
            Trace-Error "Failed to find the primary DNS server for the zone '$ExternalZoneFqdn'."
        }
        else
        {
            Trace-Warning "Failed to find the primary DNS server for the zone '$ExternalZoneFqdn'."
        }
    }

    return $PrimaryDnsServerNames
}

# This function configures all the VIPs as described in the manifest for all roles by default.
# If ThisRoleOnly parameter is set to one particular role's ID (for example, WAS), only VIPs for
# the role will be configured.
function Add-AllVIPs {

    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,
        [Parameter(Mandatory = $false)]
        [bool]
        $EnableLoadBalancerProbe = $false,
        [Parameter(Mandatory = $false)]
        [bool]
        $AddDnsRecords = $true,
        [Parameter(Mandatory = $false)]
        [string] $ThisRoleOnly = $null,
        [Parameter(Mandatory = $false)]
        [string] $ThisVipOnly = $null
    )

    $ErrorActionPreference = 'Stop'
    $VerbosePreference = "Continue"

    Trace-ECESCript "Add All VIPS" {
        $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
        $infraRole = $Parameters.Roles["Infrastructure"].PublicConfiguration
        $roleDefinition = $Parameters.Configuration.Role
        $domainRole = $Parameters.Roles["Domain"].PublicConfiguration
        $ncRoleDefinition = $Parameters.Roles["NC"].PublicConfiguration
        if ( $cloudRole.PublicInfo.Setup.CloudType -eq 'HyperConverged')
        {
            $computeRoleDefinition = $Parameters.Roles["Storage"].PublicConfiguration
        }
        else
        {
            $computeRoleDefinition = $Parameters.Roles["Compute"].PublicConfiguration
        }

        # Get the account credentials to be used.
        $securityInfo = $cloudRole.PublicInfo.SecurityInfo

        $runAsAccountId = $roleDefinition.PrivateInfo.Accounts.RunAsAccountID

        # Update case context needs to use VirtualMachines public account info
        if ($runasAccountId -eq $null)
        {
            $runAsAccountId = $Parameters.Roles["VirtualMachines"].PublicConfiguration.PublicInfo.Accounts.RunAsAccountId
        }

        $setupAccountUser = $securityInfo.DomainUsers.User | ? Role -EQ $runAsAccountId
        $setupAccountCredential = $Parameters.GetCredential($setupAccountUser.Credential)

        $networkControllerRestName = $ncRoleDefinition.PublicInfo.NetworkControllerRestIP.Name
        $domainIPMapping = Get-DomainIPMapping -Parameters $Parameters

        $domainAdminUser = $securityInfo.DomainUsers.User | ? {$_.Role -eq "DomainAdmin"}
        $domainAdminCredential = $Parameters.GetCredential($domainAdminUser.Credential)

        $isOneNode = @($Parameters.Roles["Storage"].PublicConfiguration.Nodes.Node).Count -eq 1
        $RestoreContext = Get-RestoreParameters $Parameters
        if ($isOneNode -and $RestoreContext.RestoreInprogress)
        {
            [string] $primaryDomainController = Get-AvailableADComputerName -AllADComputerName $domainIPMapping.Keys -RemoteServiceCredentials $domainAdminCredential
        }
        else
        {
            [string] $primaryDomainController = Get-AvailableADComputerName -AllADComputerName $domainIPMapping.Keys -RemoteServiceCredentials $domainAdminCredential
        }

        $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN
        $externalDomainFqdn = $domainRole.PublicInfo.DomainConfiguration.EXTERNALFQDN
        $externalDnsMachineNames  = Get-ExternalDnsMachineName -Parameters $Parameters

        $allHostNames = $computeRoleDefinition.Nodes.Node | % Name
        $clusterName = Get-ManagementClusterName $Parameters
        $allHostNames = Get-ActiveClusterNodes -Parameters $Parameters -ClusterName $clusterName
        Trace-Execution "Connecting to the network controller with REST name '$networkControllerRestName'."
        $null = Set-NCConnection -RestIP $networkControllerRestName -credential $setupAccountCredential

        $credentialParameterHashTable = @{}
        $result = $allHostNames |where { $_ -contains $env:COMPUTERNAME }
            if(!$result)
            {
                $credentialParameterHashTable = @{Credential=$setupAccountCredential}
            }
        $allVMs = Invoke-ScriptBlockWithRetries -RetryTimes 5 -RetrySleepTimeInSeconds 1 -ScriptBlock {
            $allHostNames = Get-ActiveClusterNodes -Parameters $Parameters -ClusterName $clusterName
            if($credentialParameterHashTable)
            {
                Get-VM -ComputerName $allHostNames @credentialParameterHashTable
            }
            else
            {
                Get-VM -ComputerName $allHostNames
            }
        }
        $allLBs = Get-NCLoadBalancer
        $allPublicIps = Get-NCPublicIpAddress
        $allKeys = $Parameters.Roles.Keys
        foreach($key in $allKeys)
        {
            if (((-not $ThisRoleOnly) -or ($ThisRoleOnly -eq $key) ) -and $Parameters.Roles[$key].PublicConfiguration.PublicInfo.VIPs -ne $null)
            {
                Trace-Execution "Adding all VIPs for role '$key'."
                $roleToProcess = $Parameters.Roles[$key].PublicConfiguration
                $singleVmIndexes = @{}
                try
                {
                    Trace-ECEScript "Configuring VIPs for role '$key' containing VMs: '$($constituentVMs.Name -join '', '')'." {
                        $vips = $roleToProcess.PublicInfo.VIPs.VIP
                        if ($ThisVipOnly)
                        {
                            $vips = $vips | Where-Object { $_.Id -eq $ThisVipOnly }
                        }

                        foreach ($vip in $vips)
                        {
                            $vipId = $vip.Id
                            $vipResourceId = $null
                            $vipIPv4Address = $vip.IPv4Address.Split('/')[0]
                            $VMNicName = $vip.InterfaceName
                            $targetLb = $null
                            $useStaticIp = $true

                            [string[]] $constituentVMNames = $roleToProcess.Nodes.Node | % Name
                            $constituentVMs = $allVMs | Where { $constituentVMNames.Contains($_.Name) }

                            # Support a 1-to-1 mapping of VIP to DIP.
                            # Uses the specified tag to allocate a single VM on the backend.
                            if ($vip.MapToSingleVMTag)
                            {
                                Trace-Execution "Detected MapToSingleVMTag '$($vip.MapToSingleVMTag)' on VIP."

                                $tag = $vip.MapToSingleVMTag
                                if($singleVmIndexes.ContainsKey($tag))
                                {
                                    $singleVmIndexes[$tag] += 1
                                }
                                else
                                {
                                    $singleVmIndexes.Add($tag, 0)
                                }

                                $vmIndex = $singleVmIndexes[$tag]
                                if($vmIndex -ge $constituentVMs.Count)
                                {
                                    Trace-Error "There are not enough VMs in the role '$key' to support the VIP configuration '$($vip.Id)' for tag '$tag'."
                                }

                                Trace-Execution "Pairing VIP '$($vip.Id)' with the VM index '$($vmIndex)'."
                                $constituentVMs = $constituentVMs[$vmIndex]
                            }

                            # check if the LB already exists by comparing the static IPs and publicIP resources against the provided VIP
                            $targetPublicIpObject = $allPublicIps | where-object {$_.properties.ipaddress -eq $vipIPv4Address}

                            if(-not (Get-member -Name "nextLink" -InputObject $allLBs))
                            {
                                $targetLb = $allLBs | where-object {
                                    ($_.properties.frontendipconfigurations[0].properties.privateIpaddress -eq $vipIPv4Address) -or
                                    ($targetPublicIpObject -and ($targetPublicIpObject.resourceRef -eq $_.properties.frontendipconfigurations[0].properties.publicIPAddress.resourceRef)) }
                            }
                            if ($targetLb)
                            {
                                $vipResourceId = $targetLb.resourceid
                                Trace-Execution "Found LoadBalancer using the VIP $vipIPv4Address. Using its resourceId $vipResourceId"
                            }
                            else
                            {
                                $vipResourceId = [System.Guid]::NewGuid().ToString()
                                Trace-Execution "LoadBalancer using the VIP $vipIPv4Address is not found. Use a new random GUID as new LB resourceId $vipResourceId."
                            }
                            if($targetPublicIpObject)
                            {
                                $useStaticIp = $false
                            }
                            Trace-Execution "Use static IP for the load balancer: $useStaticIp."

                            $vipName = $vip.Name

                            $isEnableOutboundNat = $true
                            if ([bool]::TryParse($vip.EnableOutboundNat, [ref] $isEnableOutboundNat) -ne $true)
                            {
                                $isEnableOutboundNat = $false
                            }

                            Trace-Execution "Processing VIP ID '$vipId' named '$vipName' with IPv4 Address '$vipIPv4Address' and EnableOutboundNat set to '$isEnableOutboundNat'."

                            if ($AddDnsRecords)
                            {
                                foreach ($endpoint in $vip.DnsEndpoints.Endpoint)
                                {
                                    $externalNetworkName = Get-NetworkNameForCluster -ClusterName "s-cluster" -NetworkName "External"
                                    if($vip.NetworkId -eq $externalNetworkName)
                                    {
                                        foreach ($externalDnsMachineName in $externalDnsMachineNames)
                                        {
                                            Add-DnsRecord -DnsName $endpoint.Path -DnsZoneName $externalDomainFqdn -IPv4Address $vipIPv4Address -DnsServerName $externalDnsMachineName -RemoteCredential $domainAdminCredential
                                        }
                                    }
                                    else
                                    {
                                        Add-DnsRecord -DnsName $endpoint.Path -DnsZoneName $domainFqdn -IPv4Address $vipIPv4Address -DnsServerName $primaryDomainController -RemoteCredential $domainAdminCredential
                                    }
                                }
                            }

                            foreach ($portMapping in $vip.PortMapping.Mapping)
                            {
                                $mappingName = $portMapping.Name
                                $frontEndPort = $portMapping.FrontEndPort
                                $backEndPort = $portMapping.BackEndPort
                                $protocol = $portMapping.Protocol
                                $probes = @()
                                if ($portMapping.Probe -ne $null -and $EnableLoadBalancerProbe)
                                {
                                    foreach ($probe in $portMapping.Probe)
                                    {
                                        $probeProtocol = $probe.Protocol
                                        $probePort = $probe.Port
                                        $probeRequestPath = $probe.RequestPath
                                        $probeIntervalInSeconds = $probe.IntervalInSeconds
                                        $probeNumberOfProbes = $probe.NumberOfProbes
                                        $probeResourceId = [system.guid]::NewGuid().ToString()

                                        # Check if the probe already exists in the LB's Probes, if so, just reuse that probe's resourceId
                                        if ($targetLb -ne $null -and $targetLb.Properties.Probes -ne $null)
                                        {
                                            foreach ($existedProbe in $targetLb.Properties.Probes)
                                            {
                                                if( $existedProbe.properties.protocol -eq $probeProtocol -and
                                                    $existedProbe.properties.port -eq $probePort -and
                                                    $existedProbe.properties.requestPath -eq $probeRequestPath -and
                                                    $existedProbe.properties.intervalInSeconds -eq $probeIntervalInSeconds -and
                                                    $existedProbe.properties.numberOfProbes -eq $probeNumberOfProbes)
                                                {
                                                    $probeResourceId = $existedProbe.resourceId
                                                    break
                                                }
                                            }
                                        }

                                        Trace-Execution "Creating in-memory LB Probe object for port mapping named '$mappingName': Protocol: $probeProtocol; Port: $probePort; RequestPath: $probeRequestPath; IntervalInSeconds: $probeIntervalInSeconds; NumberOfProbes: $probeNumberOfProbes"
                                        $probes += New-NCLoadBalancerProbeObject -resourceID $probeResourceId `
                                                            -protocol $probeProtocol `
                                                            -port $probePort `
                                                            -requestPath $probeRequestPath `
                                                            -intervalInSeconds $probeIntervalInSeconds `
                                                            -numberOfProbes $probeNumberOfProbes
                                    }
                                }

                                if ($probes.Count -eq 0)
                                {
                                    $probes = $null
                                }

                                Trace-Execution "Configuring port mapping named '$mappingName' with ports: '$frontEndPort' and '$backEndPort'; protocol: '$protocol'; and VIP IPv4 address '$vipIPv4Address'."
                                $params = @{
                                    VMPool = $constituentVMs;
                                    protocol = $protocol;
                                    frontendPort = $frontEndPort;
                                    backendPort = $backEndPort;
                                    EnableOutboundNat = $isEnableOutboundNat;
                                    Probe = $probes;
                                    LoadBalancerResourceID = $vipResourceId;
                                    VMNicName = $VMNicName;
                                }
                                if ($useStaticIp)
                                {
                                    $params.Add('Vip', $vipIpv4Address)
                                }
                                else
                                {
                                    $params.Add('PublicIpResourceRef', $targetPublicIpObject.resourceRef)
                                }
                                $null = New-LoadBalancerVIP @params
                            }

                            # A LB VIP can be created with no inbound port mappings and only OB rules
                            if ($vip.PortMapping -eq $null -and $isEnableOutboundNat)
                            {
                                Trace-Execution "Configuring outbound NAT rule with VIP IPv4 address '$vipIPv4Address', VMNicName '$VMNicName', and LoadBalancerResourceID '$vipResourceId'."
                                $params = @{
                                    VMPool = $constituentVMs;
                                    EnableOutboundNat = $isEnableOutboundNat;
                                    LoadBalancerResourceID = $vipResourceId;
                                    VMNicName = $VMNicName
                                }
                                if ($useStaticIp)
                                {
                                    $params.Add('Vip', $vipIpv4Address)
                                }
                                else
                                {
                                    $params.Add('PublicIpResourceRef', $targetPublicIpObject.resourceRef)
                                }
                                $null = New-LoadBalancerVIP @params
                            }
                        }
                    }
                }
                catch
                {
                    Write-Verbose "Failed in Add-AllVIPs. Exception $_"
                    throw
                }
            }
        }
    }
}

$global:jobArguments = @{}
$global:jobImports = @{}
$global:jobCredentials = @{}
$global:jobConfigurationNames = @{}
$global:jobAuthenticationMechanisms = @{}

function Start-ParallelWork{
        [CmdletBinding()]
        param(
            [Parameter(Mandatory = $true, ParameterSetName="Local")]
            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")]
            [ScriptBlock]
            $ScriptBlock,

            [Parameter(Mandatory = $false, ParameterSetName="Local")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")]
            [Object[]]
            $ArgumentList,

            [Parameter(Mandatory = $false, ParameterSetName="Local")]
            [bool]
            $ImportModulesInSession = $false,

            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")]
            [string[]]
            $ComputerName,

            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")]
            [string]
            $ConfigurationName,

            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")]
            [PSCredential]
            $Credential,

            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")]
            [System.Management.Automation.Runspaces.AuthenticationMechanism]
            $Authentication
        )

        $jobs = New-Object System.Collections.ArrayList
        if($PsCmdlet.ParameterSetName -eq "Local"){
            $jobs.Add((Create-LocalJob -ImportModulesInSession $ImportModulesInSession -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList)) | out-null
       }
       else{
           foreach($name in $ComputerName){
               $jobs.Add((Create-RemoteJob -ComputerName $name -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -ConfigurationName $ConfigurationName -Authentication $Authentication)) | out-null
           }
       }

       foreach($job in $jobs){
           $global:jobArguments[$job.InstanceId] = $ArgumentList
           $global:jobImports[$job.InstanceId] = $ImportModulesInSession
           $global:jobCredentials[$job.InstanceId] = $Credential
           $global:jobConfigurationNames[$job.InstanceId] = $ConfigurationName
           $global:jobAuthenticationMechanisms[$job.InstanceId] = $Authentication
       }

       return ,$jobs
}

function Create-LocalJob{
        [CmdletBinding()]
        param(
            [bool]
            $ImportModulesInSession,

            [ScriptBlock]
            $ScriptBlock,

            [Object[]]
            $ArgumentList
        )

        if($ImportModulesInSession){
           $currentFilePath = $PSCommandPath
           $currentFilePathArray = ,($currentFilePath)
           $initializationScriptContent += Create-ImportModuleString -ModulePaths $currentFilePathArray

           $modulePaths = (Get-Module).Path | where {$_ -notlike "*WindowsPowerShell*Modules*"}
           $initializationScriptContent += Create-ImportModuleString -ModulePaths $modulePaths
        }

        $initializationScript = [Scriptblock]::Create($initializationScriptContent)
        $job = Start-Job -InitializationScript $initializationScript -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList

        return $job
}

function Create-ImportModuleString {
        [CmdletBinding()]
        param(
            [Parameter(Mandatory = $true)]
            [Object[]]
            $ModulePaths
        )

        foreach($modulePath in $modulePaths){
            $importModuleString += "Import-Module $modulePath -DisableNameChecking;"
        }

        return $importModuleString
}

function Create-RemoteJob{
        [CmdletBinding()]
        param(
            [string]
            $ComputerName,

            [ScriptBlock]
            $ScriptBlock,

            [Object[]]
            $ArgumentList,

            [string]
            $ConfigurationName,

            [PSCredential]
            $Credential,

            [System.Management.Automation.Runspaces.AuthenticationMechanism]
            $Authentication
        )

        if($ConfigurationName){
            if($Authentication){
                $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -ConfigurationName $ConfigurationName -Authentication $Authentication -AsJob)
            }
            else{
                $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -ConfigurationName $ConfigurationName -AsJob)
            }
        }else{
            if($Credential){
                if($Authentication){
                    $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -Authentication $Authentication -AsJob)
                }else{
                    $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Credential $Credential -AsJob)
                }
            }else{
                if($Authentication){
                    $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -Authentication $Authentication -AsJob)
                }else{
                    $job = (Invoke-Command -ComputerName $ComputerName -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -AsJob)
                }
            }
        }

        return $job
}

function Wait-ParallelWork{
     param (
        [Parameter(Mandatory=$true)]
        [System.Collections.ArrayList]
        $Jobs,

        [Parameter(Mandatory=$false)]
        [int]
        $RetryCount = 0,

        [Parameter(Mandatory = $false)]
        [int]
        $RetrySleepTimeInSeconds = 3
     )

     $retryMap = @{}
     while($Jobs.Count -ne 0){
         $finishedJob = Wait-Job -Job $Jobs -Any

         $location = $finishedJob.Location

         $verboseBufferFromJob = $finishedJob.ChildJobs[0].Verbose
         if($verboseBufferFromJob){
             $verboseBufferFromJob | % { Trace-Execution "$location : $_" }
         }

         $warningBufferFromJob = $finishedJob.ChildJobs[0].Warning
         if($warningBufferFromJob){
             $warningBufferFromJob | % { Trace-Warning "$location : $_" }
         }

         $errorBufferFromJob = $finishedJob.ChildJobs[0].Error
         if($errorBufferFromJob){
             $errorBufferFromJob | % { Trace-Error "$location : $_" }
         }

         $informationBufferFromJob = $finishedJob.ChildJobs[0].Information
         if($informationBufferFromJob){
             $informationBufferFromJob | % { Write-Information "$location : $_" }
         }

         if($finishedJob.State -ne 'Completed'){

            $instanceId = $finishedJob.InstanceId
            if(!$retryMap.ContainsKey($instanceId)){
                $retryMap[$instanceId] = 0
            }

            $currentRetryNumber = $retryMap[$instanceId]

            if($currentRetryNumber -ge $RetryCount){
                foreach($job in $Jobs){
                    $global:jobArguments.Remove($job.InstanceId)
                    $global:jobImports.Remove($job.InstanceId)
                    $global:jobCredentials.Remove($job.InstanceId)
                    $global:jobConfigurationNames.Remove($job.InstanceId)
                    $global:jobAuthenticationMechanisms.Remove($job.InstanceId)
                }

                $Jobs | Stop-Job
                $Jobs | Remove-Job

                $failedReason = $finishedJob.ChildJobs[0].JobStateInfo.Reason

                throw "The job running on $location failed due to: $failedReason after retrying $RetryCount times"
            }
            else{
                $nextRetryNumber = $currentRetryNumber + 1
                Trace-Execution "Retrying parallel work on $location for the $nextRetryNumber time"
                Start-Sleep -s $RetrySleepTimeInSeconds

                $jobScriptBlock = [Scriptblock]::Create($finishedJob.Command)
                $jobArgumentList = $global:jobArguments[$instanceId]
                $jobImports = $global:jobImports[$instanceId]
                $jobConfiguration = $global:jobConfigurationNames[$instanceId]
                $jobCredential = $global:jobCredentials[$instanceId]
                $jobAuthenticationMechanism = $global:jobAuthenticationMechanisms[$instanceId]

                if($finishedJob.Location -eq "localhost"){
                    $job = Create-LocalJob -Scriptblock $jobScriptBlock -ArgumentList $jobArgumentList -ImportModulesInSession $jobImports
                }
                else{
                    $job = Create-RemoteJob -ComputerName $location -ScriptBlock $jobScriptBlock -ArgumentList $jobArgumentList -ConfigurationName $jobConfiguration -Credential $jobCredential -Authentication $jobAuthenticationMechanism
                }

                $newInstanceId = $job.InstanceId

                $global:jobArguments[$newInstanceId] = $jobArgumentList
                $global:jobArguments.Remove($instanceId)

                $global:jobImports[$newInstanceId] = $jobImports
                $global:jobImports.Remove($instanceId)

                $global:jobConfigurationNames[$newInstanceId] = $jobConfiguration
                $global:jobConfigurationNames.Remove($instanceId)

                $global:jobCredentials[$newInstanceId] = $jobCredential
                $global:jobCredentials.Remove($instanceId)

                $global:jobAuthenticationMechanisms[$newInstanceId] = $jobAuthenticationMechanism
                $global:jobAuthenticationMechanisms.Remove($instanceId)

                $retryMap[$newInstanceId] = $nextRetryNumber
                $retryMap.Remove($instanceId)

                $Jobs.Add($job) | out-null
            }
         }

         $finishedJob | Remove-Job
         $Jobs.Remove($finishedJob)
     }
}

function Start-ParallelWorkAndWait{
        [CmdletBinding()]
        param(
            [Parameter(Mandatory = $true, ParameterSetName="Local")]
            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")]
            [ScriptBlock]
            $ScriptBlock,

            [Parameter(Mandatory = $false, ParameterSetName="Local")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")]
            [Object[]]
            $ArgumentList,

            [Parameter(Mandatory = $false, ParameterSetName="Local")]
            [bool]
            $ImportModulesInSession = $false,

            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")]
            [string[]]
            $ComputerName,

            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")]
            [string]
            $ConfigurationName,

            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $true, ParameterSetName="RemoteWithJEA")]
            [PSCredential]
            $Credential,

            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")]
            [System.Management.Automation.Runspaces.AuthenticationMechanism]
            $Authentication = "Default",

            [Parameter(Mandatory = $false, ParameterSetName="Local")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")]
            [int]
            $RetryCount = 0,

            [Parameter(Mandatory = $false, ParameterSetName="Local")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithoutJEA")]
            [Parameter(Mandatory = $false, ParameterSetName="RemoteWithJEA")]
            [int]
            $RetrySleepTimeInSeconds = 3
        )

        if($PsCmdlet.ParameterSetName -eq "Local"){
            $jobs = Start-ParallelWork -Scriptblock $ScriptBlock -ArgumentList $ArgumentList -ImportModulesInSession $ImportModulesInSession
        }elseIf($PsCmdlet.ParameterSetName -eq "RemoteWithJEA"){
            $jobs = Start-ParallelWork -Scriptblock $ScriptBlock -ArgumentList $ArgumentList -ComputerName $ComputerName -ConfigurationName $ConfigurationName -Credential $Credential -Authentication $Authentication
        }else{
            if($Credential){
                $jobs = Start-ParallelWork -Scriptblock $ScriptBlock -ArgumentList $ArgumentList -ComputerName $ComputerName -Credential $Credential -Authentication $Authentication
            }else{
                $jobs = Start-ParallelWork -Scriptblock $ScriptBlock -ArgumentList $ArgumentList -ComputerName $ComputerName -Authentication $Authentication
            }
        }

        Wait-ParallelWork -Jobs $jobs -RetryCount $RetryCount -RetrySleepTimeInSeconds $RetrySleepTimeInSeconds
}

# Make sure the MAC addresses and IPv4Address are loaded into Parameters object from the manifest.

function Set-MacAndIPAddressSingleNode {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory=$true)]
        [PSObject]
        $Node,

        [ValidateSet('PXE','Management')]
        [string]
        $IPv4AddressSource = 'PXE'
    )

    if (-not $node.MacAddress) {
        Trace-Execution "Failed to find MAC address for node '$($node.Name)' in the manifest. Skip this"
    }
    else
    {
        $macAddress = $node.MacAddress
        Trace-Execution "Normalizing MAC address '$macAddress'."
        # Normalize MAC address to .NET canonical form, e.g., E4-1D-2D-1D-25-30.
        $node.MacAddress = $node.MacAddress.ToUpper().Replace(':', '-')
    }

    $allHostNodes = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
    
    $ipv4Address = $allHostNodes[$node.Name]
    Trace-Execution "Setting IPv4 address $ipv4Address for node $($node.Name) ."
    
    if(-not $node.IPv4Address) {
        $node | Add-Member -MemberType NoteProperty -Name "IPv4Address" -value $Ipv4Address
    } else {
        $node.IPv4Address = $ipv4Address
    }
}

function Set-MacAndIPAddress {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $true)]
        [System.Xml.XmlElement]
        $PhysicalMachinesRole,

        [ValidateSet('PXE','Management')]
        [string]
        $IPv4AddressSource = 'PXE',

        [Parameter(Mandatory=$false)]
        [System.Xml.XmlElement]
        $Node = $null
    )

    Trace-ECEScript "Setting IP addresses." {

        if ($Node)
        {
            Set-MacAndIPAddressSingleNode -Parameters $Parameters -Node $node -IPv4AddressSource $IPv4AddressSource
        }
        else
        {
            foreach ($node in $PhysicalMachinesRole.Nodes.Node)
            {
                Set-MacAndIPAddressSingleNode -Parameters $Parameters -Node $node -IPv4AddressSource $IPv4AddressSource
            }
        }
    }
}

# This function removes a specific DNS record
function Remove-DnsRecord
{
    Param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $DnsName,

        [Parameter(Mandatory=$true)]
        [string]
        $DnsZoneName,

        [Parameter(Mandatory=$false)]
        [bool]
        $IPv4Address = $false,

        [Parameter(Mandatory=$false)]
        [bool]
        $IPv6Address = $false,

        [Parameter(Mandatory=$false)]
        [bool]
        $AliasCName = $false,

        [Parameter(Mandatory=$true)]
        [string]
        $DnsServerName,

        [Parameter(Mandatory=$false)]
        [PSCredential]
        $Credential = $null
    )

    $ErrorActionPreference = 'Stop'
    return

    if($Credential)
    {
        $CimSession = New-CimSession -Credential $Credential
    }
    else
    {
        $CimSession = New-CimSession
    }

    if ($IPv4Address -eq $true)
    {
        Trace-Execution "Removing DNS resource record for name '$DnsName' with IP '$IPv4Address' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'."
        if (Get-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype A -Name $DnsName -ComputerName $DnsServerName -CimSession $CimSession -ErrorAction Ignore)
        {
            Trace-Execution "Removing existing record for '$DnsName'."
            Remove-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype A -Name $DnsName -ComputerName $DnsServerName -Force
        }
        else
        {
            Trace-Execution "Record for '$IPv4Address' (A) not present."
        }
    }

    if ($IPv6Address -eq $true)
    {
        Trace-Execution "Removing DNS resource record for name '$DnsName' with IP '$IPv6Address' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'."
        if (Get-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype AAAA -Name $DnsName -ComputerName $DnsServerName -CimSession $CimSession -ErrorAction Ignore)
        {
            Trace-Execution "Removing existing record for '$DnsName'."
            Remove-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype AAAA -Name $DnsName -ComputerName $DnsServerName -Force
        }
        else
        {
            Trace-Execution "Record for '$IPv6Address' (AAAA) not present."
        }
    }

    if ($AliasCName -eq $true)
    {
        Trace-Execution "Removing DNS resource record for name '$DnsName' with alias '$AliasCName' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'."
        if (Get-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype CName -Name $DnsName -ComputerName $DnsServerName -CimSession $CimSession -ErrorAction Ignore)
        {
            Trace-Execution "Removing existing record for '$DnsName'."
            Remove-DnsServerResourceRecord -ZoneName $DnsZoneName -RRtype CName -Name $DnsName -ComputerName $DnsServerName -Force
        }
        else
        {
            Trace-Execution "Record for '$AliasCName' (CName) not present."
        }
    }
}

function Get-DnsServerResourceRecordRemoteOrLocal {
    Param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $ZoneName,

        [Parameter(Mandatory=$true)]
        [string]
        $RRtype,

        [Parameter(Mandatory=$true)]
        [string]
        $Name,

        [Parameter(Mandatory=$false)]
        [string]
        $ComputerName,

        [Parameter(Mandatory=$false)]
        [PSCredential]
        $RemoteCredential=$null
    )
    $ErrorActionPreference = 'Stop'

    try
    {
        if ($RemoteCredential)
        {
            $computerSession = New-PsSession -ComputerName $ComputerName -Credential $RemoteCredential
            Invoke-Command -Session $computerSession -ScriptBlock { Get-DnsServerResourceRecord -ZoneName $using:ZoneName -RRtype $using:RRtype -Name $using:Name -ComputerName $using:ComputerName -ErrorAction Ignore }
        }
        else
        {
            Get-DnsServerResourceRecord -ZoneName $ZoneName -RRtype $RRtype -Name $Name -ComputerName $ComputerName -ErrorAction Ignore
        }
    }
    finally
    {
        if ($computerSession)
        {
            Remove-PSSession $computerSession
        }
    }
}

function Remove-DnsServerResourceRecordRemoteOrLocal {
    Param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $ZoneName,

        [Parameter(Mandatory=$true)]
        [string]
        $RRtype,

        [Parameter(Mandatory=$true)]
        [string]
        $Name,

        [Parameter(Mandatory=$false)]
        [string]
        $ComputerName,

        [Parameter(Mandatory=$false)]
        [PSCredential]
        $RemoteCredential=$null
    )
    $ErrorActionPreference = 'Stop'

    try
    {
        if ($RemoteCredential)
        {
            $computerSession = New-PsSession -ComputerName $ComputerName -Credential $RemoteCredential
            Invoke-Command -Session $computerSession -ScriptBlock { Remove-DnsServerResourceRecord -ZoneName $using:ZoneName -RRtype $using:RRtype -Name $using:Name -ComputerName $using:ComputerName -Force }
        }
        else
        {
            Remove-DnsServerResourceRecord -ZoneName $ZoneName -RRtype $RRtype -Name $Name -ComputerName $ComputerName -Force
        }
    }
    finally
    {
        if ($computerSession)
        {
            Remove-PSSession $computerSession
        }
    }
}

function Add-DnsServerResourceRecordCNameRemoteOrLocal {
    Param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $ZoneName,

        [Parameter(Mandatory=$true)]
        [string]
        $Name,

        [Parameter(Mandatory=$true)]
        [string]
        $HostNameAlias,

        [Parameter(Mandatory=$false)]
        [string]
        $ComputerName,

        [Parameter(Mandatory=$false)]
        [PSCredential]
        $RemoteCredential=$null
    )
    $ErrorActionPreference = 'Stop'

    try
    {
        if ($RemoteCredential)
        {
            $computerSession = New-PsSession -ComputerName $ComputerName -Credential $RemoteCredential
            Invoke-Command -Session $computerSession -ScriptBlock { Add-DnsServerResourceRecordCName -ZoneName $using:ZoneName -Name $using:Name -HostNameAlias $using:HostNameAlias -ComputerName $using:ComputerName }
        }
        else
        {
            Add-DnsServerResourceRecordCName -ZoneName $ZoneName -Name $Name -HostNameAlias $HostNameAlias -ComputerName $DnsServerName
        }
    }
    finally
    {
        if ($computerSession)
        {
            Remove-PSSession $computerSession
        }
    }
}

function Add-DnsServerResourceRecordRemoteOrLocal {
    Param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $ZoneName,

        [Parameter(Mandatory=$true)]
        [string]
        $Name,

        [Parameter(Mandatory=$false)]
        [switch]
        $IsIPv6,

        [Parameter(Mandatory=$true)]
        [string]
        $IPAddress,

        [Parameter(Mandatory=$false)]
        [string]
        $ComputerName,

        [Parameter(Mandatory=$false)]
        [PSCredential]
        $RemoteCredential=$null
    )
    $ErrorActionPreference = 'Stop'

    try
    {
        if ($RemoteCredential)
        {
            $computerSession = New-PsSession -ComputerName $ComputerName -Credential $RemoteCredential

            if ($IsIPv6)
            {
                Invoke-Command -Session $computerSession -ScriptBlock { Add-DnsServerResourceRecord -ZoneName $using:ZoneName -AAAA -Name $using:Name -IPv6Address $using:IPAddress -ComputerName $using:ComputerName }
            }
            else
            {
                Invoke-Command -Session $computerSession -ScriptBlock { Add-DnsServerResourceRecord -ZoneName $using:ZoneName -A -Name $using:Name -IPv4Address $using:IPAddress -ComputerName $using:ComputerName }
            }
        }
        else
        {
            if ($IsIPv6)
            {
                Add-DnsServerResourceRecord -ZoneName $ZoneName -AAAA -Name $Name -IPv6Address $IPAddress -ComputerName $ComputerName
            }
            else
            {
                Add-DnsServerResourceRecord -ZoneName $ZoneName -A -Name $Name -IPv4Address $IPAddress -ComputerName $ComputerName
            }
        }
    }
    finally
    {
        if ($computerSession)
        {
            Remove-PSSession $computerSession
        }
    }
}

# This function creates the DNS record for the given name.
function Add-DnsRecord
{
    Param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $DnsName,

        [Parameter(Mandatory=$true)]
        [string]
        $DnsZoneName,

        [Parameter(Mandatory=$false)]
        [string]
        $IPv4Address,

        [Parameter(Mandatory=$false)]
        [string]
        $IPv6Address,

        [Parameter(Mandatory=$false)]
        [string]
        $AliasCName,

        [Parameter(Mandatory=$true)]
        [string]
        $DnsServerName,

        [Parameter(Mandatory=$false)]
        [bool]
        $RemovePreviousRecord = $true,

        [Parameter(Mandatory=$false)]
        [PSCredential]
        $RemoteCredential=$null
    )

    $ErrorActionPreference = 'Stop'

    Trace-ECEScript "Adding DNS record $DnsName" {
        if ($IPv4Address)
        {
            Trace-Execution "Adding DNS resource record for name '$DnsName' with IP '$IPv4Address' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'."
            if (Get-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype A -Name $DnsName -ComputerName $DnsServerName -ErrorAction Ignore -RemoteCredential $RemoteCredential)
            {
                if ($RemovePreviousRecord -eq $true)
                {
                    Trace-Execution "Removing existing record for '$DnsName'."
                    Remove-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype A -Name $DnsName -ComputerName $DnsServerName -RemoteCredential $RemoteCredential
                }
            }
            Trace-Execution "Adding record for '$DnsName'."

            $RetryTimes = 3
            for ($retry = 0; $retry -lt $RetryTimes; $retry ++)
            {
                try
                {
                    Add-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -Name $DnsName -IPAddress $IpV4Address -ComputerName $DnsServerName -RemoteCredential $RemoteCredential
                    break;
                }
                catch
                {
                    if ($retry -lt ($RetryTimes - 1))
                    {
                        continue
                    }
                    else
                    {
                        throw
                    }
                }
            }
        }

        if ($IPv6Address)
        {
            Trace-Execution "Adding DNS resource record for name '$DnsName' with IP '$IPv6Address' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'."
            if (Get-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype AAAA -Name $DnsName -ComputerName $DnsServerName -ErrorAction Ignore -RemoteCredential $RemoteCredential)
            {
                if ($RemovePreviousRecord -eq $true)
                {
                    Trace-Execution "Removing existing record for '$DnsName'."
                    Remove-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype AAAA -Name $DnsName -ComputerName $DnsServerName -RemoteCredential $RemoteCredential
                }
            }
            Trace-Execution "Adding record for '$DnsName'."
            Add-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -Name $DnsName -IsIPv6 -IPAddress $IPv6Address -ComputerName $DnsServerName -RemoteCredential $RemoteCredential
        }

        if ($AliasCName)
        {
            Trace-Execution "Adding DNS resource record for name '$DnsName' with alias '$AliasCName' under zone name '$DnsZoneName' on the domain controller '$DnsServerName'."
            if (Get-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype CName -Name $DnsName -ComputerName $DnsServerName -ErrorAction Ignore -RemoteCredential $RemoteCredential)
            {
                if ($RemovePreviousRecord -eq $true)
                {
                    Trace-Execution "Removing existing record for '$DnsName'."
                    Remove-DnsServerResourceRecordRemoteOrLocal -ZoneName $DnsZoneName -RRtype CName -Name $DnsName -ComputerName $DnsServerName -RemoteCredential $RemoteCredential
                }
            }
            Trace-Execution "Adding record for '$DnsName'."
            Add-DnsServerResourceRecordCNameRemoteOrLocal -ZoneName $DnsZoneName -Name $DnsName -HostNameAlias $AliasCName -ComputerName $DnsServerName -RemoteCredential $RemoteCredential
        }
    }
}

# adds PTR records for all a-type records in external zone FQDN
function Add-DnsExternalPtrRecords {
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )


    $primaryDnsNames = Get-ExternalDnsMachineName -Parameters $Parameters -ErrorOnFail $false

    if ($primaryDnsNames.Count -eq 0)
    {
        Trace-Error "No Primary DNS server found. Selecting default all WASP machines"
        $waspRole = $Parameters.Roles["WASPUBLIC"].PublicConfiguration
        $primaryDnsNames   = [string[]]$waspRole.Nodes.Node.Name # <VM Name="[PREFIX]-WASP##" />
    }

    foreach($primaryDnsName in $primaryDnsNames)
    {
        $DnsExternalZoneFqdn = $Parameters.Roles["ExternalDNS"].PublicConfiguration.PublicInfo.DnsConfiguration.ExternalZoneFqdn
        $DnsResourceRecords = @(Get-DnsServerResourceRecord -ZoneName $DnsExternalZoneFqdn -computerName $primaryDnsName)
        $ExternalZoneTTLInHours = $Parameters.Roles["ExternalDNS"].PublicConfiguration.PublicInfo.DnsConfiguration.ExternalZoneTTLInHours
        $timeToLive = (New-TimeSpan -Hours $ExternalZoneTTLInHours)

        Trace-Execution "Getting Dns Resource Records for zone $DnsExternalZoneFqdn"

        foreach ($DnsResourceRecord in $DnsResourceRecords | where {$_.RecordType -eq 'A'})
        {
            Trace-Execution "Processing A record $($DnsResourceRecord.RecordData)"
            $IPv4 = $DnsResourceRecord.RecordData.IPv4address.tostring().split('.')
            $zoneName = $IPv4[0] + ".in-addr.arpa"
            $PtrName = $IPv4[3..1] -join "."
            $ptrDomainName = $DnsResourceRecord.HostName + '.' + $DnsExternalZoneFqdn

            Invoke-ScriptBlockWithRetries -RetryTimes 3 -RetrySleepTimeInSeconds 1 -ScriptBlock {
                $serverZone = Get-DnsServerZone -Name $zoneName -computerName $primaryDnsName -ErrorAction SilentlyContinue

                if(-not $serverZone)
                {
                    Throw "Failed to add PTR record $PtrName, could not find zoneName: $zoneName"
                }

                $ptrRecord = Get-DnsServerResourceRecord -ZoneName $zoneName -name $PtrName -computerName $primaryDnsName -ErrorAction SilentlyContinue

                if (-not $ptrRecord)
                {
                    Trace-Execution "Adding PtrDomain $ptrDomainName, PtrRecordName $PtrName for zone $zoneName"
                    try
                    {
                        Add-DnsServerResourceRecordPtr -computerName $primaryDnsName -ZoneName $zoneName -PtrDomainName $ptrDomainName -name $PtrName -TimeToLive $timeToLive
                    }
                    catch
                    {
                        Trace-Warning "Trying to handle Exception: $_"
                        $ptrZoneResourceRecord = @(Get-DnsServerResourceRecord -ZoneName $zoneName -computerName $primaryDnsName)
                        # check if there is a delegate zone blocking
                        if ($ptrZoneResourceRecord | where {$_.RecordType -eq 'NS' -and $PtrName -match $_.HostName.replace(".", "\.")})
                        {
                            Trace-Execution "Did not add $PtrName to zone $zoneName because a delegate zone $ptrZoneNsRecord exists"
                            continue
                        }
                        Trace-Warning "Failed to add ptrRecord $PtrName to zone $zoneName. No NS zone blocking ptrRecord found."
                    }
                }

                # Update the TTL for PTR record
                $oldRecordPTR = Get-DnsServerResourceRecord -ZoneName $zoneName -name $PtrName -computerName $primaryDnsName -ErrorAction SilentlyContinue
                if ($oldRecordPTR -and $oldRecordPTR.TimeToLive -ne $timeToLive)
                {
                    # Calling Add on an existing record is safe. It overwrites the existing record if there is one.
                    Trace-Execution "Updating the TTL of $PtrName on zone $zoneName to $timeToLive."
                    Add-DnsServerResourceRecordPtr -ZoneName $zoneName -PtrDomainName $ptrDomainName -name $PtrName -TimeToLive $timeToLive -ComputerName $primaryDnsName
                }

                # Update the TTL for A record
                $oldRecordA = Get-DnsServerResourceRecord -ZoneName $DnsExternalZoneFqdn -name $DnsResourceRecord.HostName -ComputerName $primaryDnsName -ErrorAction SilentlyContinue
                if ($oldRecordA -and $oldRecordA.TimeToLive -ne $timeToLive)
                {
                    # Calling Add on an existing record is safe. It overwrites the existing record if there is one.
                    Trace-Execution "Updating the TTL of $($DnsResourceRecord.HostName) on zone $DnsExternalZoneFqdn to $timeToLive."
                    Add-DnsServerResourceRecordA -ZoneName $DnsExternalZoneFqdn -name $DnsResourceRecord.HostName -TimeToLive $timeToLive -IPv4Address $DnsResourceRecord.RecordData.IPv4Address -ComputerName $primaryDnsName
                }
            }
        }
    }
}

# Returns the offline domain join blob used for unattended domain join.
function Get-OfflineDjoinBlob {

    Param (
        [Parameter(Mandatory=$true)]
        [string[]]
        $ComputerName,

        [Parameter(Mandatory=$true)]
        [string]
        $OuPath,

        [string[]]
        $DomainFqdn = $env:USERDNSDOMAIN,

        [Parameter(Mandatory=$false)]
        [PSCredential]
        $DomainJoinCredential=$null
    )

    foreach ($eachComputer in $ComputerName) {
        $tempFilePath = [IO.Path]::GetTempFileName()
        Trace-Execution "Retrieving odj blob for host '$eachComputer' using the temporary file '$tempFilePath'."


        function Join-DomainOffline
        {
            param($DomainFqdn, $ComputerName, $tempFilePath, $OuPath)

            # Entirely arbitrary number of retries until we understand the occasional failure.
            $counter = 6
            do
            {
                djoin.exe /provision /domain $DomainFqdn /machine $ComputerName /machineOU $OuPath /savefile $tempFilePath /reuse
                if (0 -eq $LASTEXITCODE)
                {
                    break
                }
                else
                {
                    Write-Warning "Failed to execute DJOIN.exe, retrying."
                    Start-Sleep -Seconds 5
                }
                $counter--
            }
            while ($counter -ge 0)
        }

        if ($DomainJoinCredential)
        {
            $pssession = New-PSSession -Credential $DomainJoinCredential -Authentication Credssp
            try
            {
                $cmdOutput = Invoke-Command `
                    -ArgumentList $DomainFqdn, $eachComputer, $tempFilePath, $OuPath `
                    -ScriptBlock ${function:Join-DomainOffline} `
                    -Session $pssession
            }
            finally
            {
                $pssession | Remove-PsSession
            }
        }
        else
        {
            $cmdOutput = Join-DomainOffline -DomainFqdn $DomainFqdn -ComputerName $eachComputer -tempFilePath $tempFilePath -OuPath $OuPath
        }

        [string] $eachOdjBlob = Get-Content $tempFilePath -Raw
        if (-not $eachOdjBlob) {
            Trace-Error "Failed to retrieve the blob. Ouput of djoin.exe: '$cmdOutput'."
        }
        # Return the blob for each input.
        $eachOdjBlob = ($eachOdjBlob.Split(0))[0] # Remove the end of line null character (results in XML parse errors).
        Write-Output $eachOdjBlob
    }

    # Force a synchronization of domain controllers
   # (Get-ADDomainController -Filter *).Name | Foreach-Object {repadmin /syncall $_ (Get-ADDomain).DistinguishedName /e /A | Out-Null}
}

function Get-BareMetalCredential
{
    [CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $cloudRole = $Parameters.Roles['Cloud'].PublicConfiguration

    # Account info
    $securityInfo = $cloudRole.PublicInfo.SecurityInfo
    $bareMetalUser = $securityInfo.HardwareUsers.User | Where-Object -Property Role -EQ 'BareMetalAdmin'
    $bareMetalCredential = $Parameters.GetCredential($bareMetalUser.Credential)

    return $bareMetalCredential
}

# Returns the parent action plan instance ID.
function Get-ActionPlanInstanceID
{
    [CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $runtimeParameters = $Parameters.RunInformation['RuntimeParameter']
    if ($runtimeParameters -and $runtimeParameters.ContainsKey('ParentActionPlanInstanceIDToken'))
    {
        [Guid]$actionPlanInstanceId = $runtimeParameters['ParentActionPlanInstanceIDToken']
    }
    else
    {
        $actionPlanInstanceId = [Guid]::Empty
    }

    return $actionPlanInstanceId
}

# Returns a common location to place test logs.
# If the value found is $null, return $null. Otherwise, make sure the folder exist.
function Get-TestLogsPath
{
    [CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [switch]
        $RemotePath
    )

    if ($RemotePath)
    {
        $testLogsPath = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.TestInfo.TestLogs.RemotePath
    }
    else
    {
        $testLogsPath = $ExecutionContext.InvokeCommand.ExpandString($Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.TestInfo.TestLogs.LocalPath)

        if (-not $testLogsPath)
        {
            # Not localizing as this would be a CustomerConfig template error
            Trace-Error "Path missing in CustomerConfig: Cloud\PublicInfo\TestInfo\TestLogs\LocalPath"
        }
    }

    if (-not $testLogsPath)
    {
        # This is a Deployment and a share is not available yet, so keep the logs on their local path
        return $null
    }

    $testLogsPath = $ExecutionContext.InvokeCommand.ExpandString($testLogsPath)

    if (-not (Test-Path -Path $testLogsPath -PathType Container))
    {
        Trace-Execution "Creating folder $testLogsPath"
        New-Item -Path $testLogsPath -ItemType Directory | Out-Null
    }

    return $testLogsPath
}

# This function runs the specified test (local or remote) using Pester framework.
function Start-Test
{
    Param
    (
        [Parameter(Mandatory=$true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        # Specify the Script parameter to be passed to Pester.
        # For Example, @{'Path' = "d:\Tests"; 'Parameters' = @{'ComputerName' = "r1s8f33-SUS01"}
        # where 'Path' is the path to the test files, and 'Parameters' is a hashtable of parameters to be passed to the test.
        [Parameter(Mandatory=$true)]
        [hashtable]
        $Script,

        [string[]]
        $ComputerName = $env:computername,

        [PSCredential]
        $Credential,

        [string]
        $TestName = "Validate-Role"
    )

    $ErrorActionPreference = 'Stop'

    $testPath = $Script.Get_Item("Path")
    if (-not $testPath)
    {
        Trace-Error ($LocalizedStrings.KeyValueNotProvided -f "Script","Path")
    }

    $roleName = $Parameters.Configuration.Role.Id
    $logPath = Get-TestLogsPath -Parameters $Parameters

    foreach ($computer in $ComputerName)
    {
        # Run test and create log locally on the machine
        Trace-Execution ($LocalizedStrings.ValidationTestStarting -f $roleName,$computer)

        # To avoid clash with parallel running test generate unique Guid
        $uniqueSubString = [Guid]::NewGuid().toString().subString(0,4)

        $outputFileName = "TestResults_$($computer)_$($roleName)_$($uniqueSubString)_$((Get-Date).ToString("yyyy-MM-dd-HH-mm-ss")).xml"
        $outputFile = "$logPath\$outputFileName"

        $pesterParameters = @{Script=$Script;TestName=$TestName;OutputFile=$outputFile;OutputFormat="NUnitXml";PassThru=$true;Quiet=$true}
        $remoteScript = {
            param(
                $LogPath,
                $PesterParameters
            )

            Import-Module pester -RequiredVersion "3.4.0"
            if (-not (Test-Path -Path $LogPath -PathType Container))
            {
                New-Item -Path $LogPath -ItemType Directory | Out-Null
            }

            Invoke-Pester @PesterParameters
        }

        if ($Credential)
        {
            $result = Invoke-Command -ScriptBlock $remoteScript -ArgumentList $logPath,$pesterParameters -ComputerName $computer -Credential $Credential
        }
        elseif ($computer -eq $env:computername)
        {
            Import-Module pester -RequiredVersion "3.4.0"
            $result = Invoke-Pester @pesterParameters
        }
        else
        {
            $result = Invoke-Command -ScriptBlock $remoteScript -ArgumentList $logPath,$pesterParameters -ComputerName $computer
        }

        # Copy log to remote location
        $remoteLogPath = Get-TestLogsPath -Parameters $Parameters -RemotePath
        $remoteOutputFile = "\\$computer\$($outputFile.Replace(':','$'))"

        # For Deployment action, $remoteLogPath will be $null.
        # In this case, don't move the test logs, and leave them on the test machine.
        if ($remoteLogPath)
        {
            # copy the pester xml result file to a remote share (or some other local path)
            $copyResult = Copy-Item -Path $remoteOutputFile -Destination $remoteLogPath -Force -PassThru
            if ($copyResult)
            {
                $fileToRemove = $remoteOutputFile
                $remoteOutputFile = "$remoteLogPath\$outputFileName"

                # remove the original pester output file to avoid wasting space
                Remove-Item -Path $fileToRemove -Force -ErrorAction SilentlyContinue | Out-Null
            }
        }

        # Print passing tests
        foreach ($testResult in ($result.TestResult | Where-Object {$_.Passed}))
        {
            Trace-Execution "Test $($testResult.Result): $($testResult.Name)"
        }

        if ($result.FailedCount)
        {
            $message = $LocalizedStrings.ValidationTestFailed -f $roleName,$computer
            $message += "`r`n$($LocalizedStrings.TestResultsLocation -f $remoteOutputFile)"

            # Include failed tests in the message
            foreach ($testResult in ($result.TestResult | Where-Object {$_.Passed -ne $true}))
            {
                $message += "`r`nTest $($testResult.Result): $($testResult.Name)"
            }

            Trace-Error $message
        }
        elseif ($result.PassedCount -eq 0)
        {
            $message = $LocalizedStrings.ValidationTestNotAvailable -f $roleName,$computer
            $message += "`r`n$($LocalizedStrings.TestResultsLocation -f $remoteOutputFile)"
            Trace-Error $message
        }
        else
        {
            Trace-Execution ($LocalizedStrings.ValidationTestFinished -f $roleName,$computer)
            Trace-Execution ($LocalizedStrings.TestResultsLocation -f $remoteOutputFile)
        }
    }
}

function Get-AvailableServer
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory=$true)]
        [string[]]
        $Server
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    $Server | % {

        $null = Test-WSMan -ComputerName $_ -ErrorAction SilentlyContinue -ErrorVariable TestWSMANError

        if(-not $testWsmanError)
        {
            $availableServer += @($_)
        }
        else
        {
            Write-Warning ($LocalizedStrings.CouldNotConnectToServerWarning -f $_)
        }
    }

    return $availableServer
}

function Start-CloudCluster
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory=$true)]
        [string]
        $ClusterName,

        [Parameter(Mandatory=$true)]
        [string[]]
        $ClusterNodeNames,

        [Parameter(Mandatory=$true)]
        [string[]]
        $ResourceName,

        [int]
        $StartClusterTimeoutSec = 180,

        [Parameter(Mandatory=$true)]
        [PSCredential]
        $Credential
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    # Set Cluster service to automatic if its not already.
    Trace-Execution ($LocalizedStrings.SetClusterServiceToAutomatic -f @($ClusterName, ($ClusterNodeNames -join ', ')))

    Invoke-Command -ComputerName $ClusterNodeNames -Credential $Credential {

        Set-Service -Name ClusSvc -StartupType Automatic -ErrorAction Stop
    }

    # Check if Cluster is already up.
    Trace-Execution ($LocalizedStrings.VerifyingCluster -f @($ClusterName, ($ResourceName -join ', ')))

    $clusterAlreadyAvailable = $true

    $clusterObject = $null
    $clusterObject = Get-Cluster -Name $ClusterName -ErrorAction SilentlyContinue -Verbose:$false

    if(-not $clusterObject)
    {
        $clusterAlreadyAvailable = $false
    }
    else
    {
        $clusterAlreadyAvailable = Test-ClusterResourceExist -ClusterObject $clusterObject -ResourceName $ResourceName
    }

    if(-not $clusterAlreadyAvailable)
    {
        Trace-Execution ($LocalizedStrings.ClusterNotReadyRestartingService -f @($ClusterName, ($ClusterNodeNames -join ', ')))

        # Starting all the cluster services starts the cluster when quorum is reached.
        Invoke-Command -ComputerName $ClusterNodeNames -Credential $Credential {

            Restart-Service -Name ClusSvc -ErrorAction Stop
        }

        Trace-Execution ($LocalizedStrings.CheckingClusterResources -f @($ClusterName,  ($ResourceName -join ', ')))

        $result = Wait-Result -TimeOut $startClusterTimeoutSec -Interval 10 -ValidationScript {

            Get-Cluster -Name $ClusterName
        }

        if(-not $result)
        {
            throw ($LocalizedStrings.UnableToGetClusterTimeout -f @($ClusterName, $startClusterTimeoutSec))
        }

        $clusterObject = Get-Cluster -Name $ClusterName

        $result = Wait-Result -TimeOut $startClusterTimeoutSec -Interval 10 `
            -ValidationScript { Test-ClusterResourceExist -ClusterObject $clusterObject -ResourceName $ResourceName }

        if(-not $result)
        {
            throw ($LocalizedStrings.ClusterResourcesTimeout -f @($($ResourceName -join ', '), $StartClusterTimeoutSec))
        }

        Trace-Execution ($LocalizedStrings.ClusterStartedSuccessfully -f $ClusterName)
    }
    else
    {
        Trace-Execution ($LocalizedStrings.ClusterAlreadyStarted -f $ClusterName)
    }
}

function Test-ClusterResourceExist
{
    [CmdletBinding()]
    [OutputType([System.Boolean])]
    Param (
        [Parameter(Mandatory=$true)]
        [Object]
        $ClusterObject,

        [Parameter(Mandatory=$true)]
        [string[]]
        $ResourceName
    )

    $clusterResources = Invoke-ScriptBlockWithRetries -RetryTimes 5 -RetrySleepTimeInSeconds 1 -ScriptBlock {Get-ClusterResource -Cluster $ClusterObject}
    $resourcesresult = $clusterResources | ? Name -In $ResourceName

    # Resources not found yet
    if(-not $resourcesresult)
    {
        return $false
    }
    else
    {
        $onlineResult = $resourcesresult | ? State -NE 'Online'

        # One or more Resources not online yet
        if($onlineResult)
        {
            return $false
        }
    }

    return $true
}

function Stop-CloudCluster
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory=$true)]
        [string]
        $ClusterName,

        [Parameter(Mandatory=$true)]
        [string[]]
        $ClusterNodeNames,

        [Parameter(Mandatory=$true)]
        [PSCredential]
        $Credential
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    $availableServer = Get-AvailableServer -Server $ClusterNodeNames

    if($availableServer)
    {
        Trace-Execution ($LocalizedStrings.GettingCluster -f $ClusterName)

        $clusterObject = $null
        $clusterObject = Get-Cluster -Name $ClusterName -ErrorAction SilentlyContinue -ErrorVariable clusterError -Verbose:$false

        if($clusterObject)
        {
            Trace-Execution ($LocalizedStrings.StoppingCluster -f $ClusterName)

            Stop-Cluster -Cluster $clusterObject -Force
        }
        else
        {
            Trace-Execution $clusterError
            Trace-Warning ($LocalizedStrings.UnableToGetCluster -f $ClusterName)
        }
    }
}

Function New-CimSessionVerify
{
    param(
        [string]$ComputerName,

        [PSCredential] $Credential,

        [Int]$Retries = 20
    )
    $tracingStarted = $false
    do {

        $cimSession = New-CimSession -ComputerName $computerName -Credential $Credential -ErrorAction Ignore
        if ($cimSession -ne $null)
        {
            Trace-Execution "CimSession established"
            break
        }

        $traceName = "$env:systemdrive\CimSessionTrace$Retries.etl"
        if ($tracingStarted -eq $true)
        {
            Trace-Execution "Stopping CimSession tracing Iteration $Retries ($traceName)"
            # stop any previous tracing
            $log = Invoke-Command -ScriptBlock {netsh trace stop}
            Trace-Execution "Log $log"
            $tracingStarted = $false
        }

        $Retries--

        # stop any previous tracing
        $log = Invoke-Command -ScriptBlock {netsh trace stop}
        Trace-Execution "Log $log"

        # start further tracing
        Trace-Execution "Start Tracing for CimSession, iteration $Retries ($traceName)"
        $log = Invoke-Command -ScriptBlock { `
            netsh trace start scenario=virtualization provider=microsoft-windows-tcpip level=5 provider=microsoft-windows-dns-client capture=yes capturetype=both report=disabled tracefile=$traceName ov=yes }
        Trace-Execution "Log $log"
        $tracingStarted = $true

        #verify we have DNS resolution
        Trace-Execution "Verify DNS resolution to $computerName"
        $dns  = Resolve-DnsName $computerName -ErrorAction Ignore
        if ($dns)
        {
            Trace-Execution "Resolve-DnsName returned"
            Trace-Execution ($dns | out-string)
        }

        # verify that we have connectivity to DC
        Trace-Execution "Verify IP connectivity to $computerName"
        $tnc = Test-NetConnection -ComputerName $computerName -Verbose -ErrorAction Ignore
        if ($tnc)
        {
            Trace-Execution "Test-NetConnection returned"
            Trace-Execution ($tnc | out-string)
        }

        # verify that we have WASMAN connectivity to DC
        Trace-Execution "Verify WSMAN connectivity to $computerName"
        $wsman = Test-WSMan -ComputerName $computerName -Verbose -ErrorAction Ignore
        if ($wsman)
        {
            Trace-Execution "Test-WSMan returned"
            Trace-Execution ($wsman | out-string)
        }

        Trace-Execution "Waiting 15 secs"
        Sleep 15
    } while ($Retries -gt 0)

    if ($tracingStarted -eq $true)
    {
        Trace-Execution "Stopping CimSession tracing Iteration $Retries ($traceName)"
        # stop any previous tracing
        $log = Invoke-Command -ScriptBlock {netsh trace stop}
        Trace-Execution "Log $log"
        $tracingStarted = $false
    }
    if ($cimSession -eq $null)
    {
        Trace-Error "No CIM Session available on $computerName"
        throw
    }

    return $cimSession
}

Function Set-DNSForwarder
{
    [CmdletBinding()]
    param(
        [String[]]$IPAddress,

        [String]
        $ComputerName,

        [PSCredential]
        $Credential
    )

    # Setting IP DNS forwarders
    Trace-Execution "Setting IP DNS forwarders on $computerName to '$IPAddress'"

    $cimSession = New-CimSessionVerify -ComputerName $computerName -Credential $Credential
    if ($cimSession)
    {
        foreach($dnsIP in $IPAddress)
        {
            Trace-Execution "Start: Set the IP DNS forwarder '$dnsIP'."
            $ip = [System.Net.IPAddress]::Parse($dnsIP)
            $addressExist = (Get-DnsServerForwarder -CimSession $cimSession).IPAddress | where {$_.Address -eq  $ip.Address}
            if ($addressExist)
            {
                Trace-Execution "Address $dnsIP existed, removing"
                Remove-DnsServerForwarder -IPAddress $dnsIP -CimSession $cimSession -Force
            }
            Trace-Execution "Adding address $dnsIP"
            Add-DnsServerForwarder -IPAddress $dnsIP -CimSession $cimSession
            Trace-Execution "End: Set the IP DNS forwarder '$dnsIP'."
        }

        Remove-CimSession $cimSession
    }
    else
    {
        Trace-Error "No CIM Session available on $computerName"
    }
}

# Creates users, groups, and group membership in the AD as defined in the SecurityInfo section.
function Add-AccountMemberships {

    [CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $true)]
        [String]
        $PrimaryDomainControllerName,

        [Bool]
        $IsIdempotentRun=$false
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
    $domainRole = $Parameters.Roles["Domain"].PublicConfiguration

    $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN
    $domainIPMapping = Get-DomainIPMapping -Parameters $Parameters
    $primaryDomainControllerIP = $domainIPMapping[$PrimaryDomainControllerName]

    [bool] $isHyperConverged = $cloudRole.PublicInfo.Setup.CloudType -eq "HyperConverged"
    $SecurityInfoRoles = @()
    $domainGroups = @()
    $domainUsers = @()
    $serviceAccounts = @()

    $SecurityInfoRoles = $Parameters.Roles.GetEnumerator() |
            ? { $Parameters.Roles[$_.Name].PublicConfiguration.PublicInfo.SecurityInfo } |
            % { $_.Name }

    foreach ($role in $SecurityInfoRoles)
    {
        $securityInfo = $Parameters.Roles[$role].PublicConfiguration.PublicInfo.SecurityInfo
        $domainGroups += $securityInfo.DomainGroups.Group
        $domainUsers += $securityInfo.DomainUsers.User
        $serviceAccounts += $securityInfo.ServiceAccounts.Account
    }

    #Removing empty lines from the array
    $domainGroups = $domainGroups | ? {$_}
    $domainUsers = $domainUsers | ? {$_}
    $serviceAccounts = $serviceAccounts | ? {$_}

    # Domain Admin Credentials
    $domainAdminUser = $cloudRole.PublicInfo.SecurityInfo.DomainUsers.User | ? Role -EQ "DomainAdmin"
    $domainCredential = $Parameters.GetCredential($domainAdminUser.Credential)
    $primaryDomainControllerSession = New-PSSession -ComputerName $primaryDomainControllerIP -Credential $domainCredential -Authentication Credssp

    $secretAgreementParam = Get-Content -Encoding Byte "$PSScriptRoot\param3072"

    # Get update version in case it is an update
    # No need to change KDS key len for update
    $updateVersion = Get-InProgressUpdateVersion -Parameters $Parameters
    if($updateVersion -eq $null)
    {
        Trace-Execution "Updating KDS..."
        Invoke-Command -Session $primaryDomainControllerSession -ScriptBlock {
            Trace-Execution "Retrieving the current KDS config."
            $kdsConfig = Get-KdsConfiguration
            if (($kdsConfig.SecretAgreementPublicKeyLength -eq 3072) -and ($kdsConfig.SecretAgreementAlgorithm -eq 'DH'))
            {
                Trace-Execution "KDS key Length is already 3072: no need to take further actions."
                return
            }

            Trace-Execution "Changing KDS key length to 3072."
            $kdsConfig = Set-KdsConfiguration -SecretAgreementPublicKeyLength 3072 -SecretAgreementParameters $using:secretAgreementParam -SecretAgreementAlgorithm DH
            if (($kdsConfig.SecretAgreementPublicKeyLength -ne 3072) -or ($kdsConfig.SecretAgreementAlgorithm -ne 'DH'))
            {
                Trace-Error "Failed to update KDS: SecretAgreementPublicKeyLength $($kdsConfig.SecretAgreementPublicKeyLength), SecretAgreementAlgorithm $($kdsConfig.SecretAgreementAlgorithm)"
            }

            # Added so that domain service accounts can be created from all domain controllers (executed as domain admin).
            Trace-Execution "Changed KDS key length to 3072, now adding KDS root key."
            $rootKey = Add-KdsRootKey -EffectiveTime ((Get-Date).AddHours(-10))
            Trace-Execution "New KDS root key GUID is $($rootKey.Guid)"
        }
    }

    Trace-Execution "Processing domain groups defined in the manifest."
    # Decoupling membership and new domian group creation
    #creating domain grous.
    Trace-Execution "Adding new user groups"
    foreach ($domainGroup in $domainGroups) {
        if($isHyperConverged -and $domainGroup.Id -eq "ComputeNodes")
        {
            Trace-Execution "Skipping Group $($domainGroup.Id) as its a HyperConverged topology."
        }
        else
        {
            if ($domainGroup.SkipCreation -and $domainGroup.SkipCreation -eq "True")
            {
                Trace-Execution "Skipping Group $($domainGroup.Id) creation."
            }
            else
            {
                Trace-Execution "Adding Group $($domainGroup.Id)"
                $null = New-UserGroup -ADRemoteSession $primaryDomainControllerSession -Groups $domainGroup -IsIdempotentRun $IsIdempotentRun
            }
        }
    }

    #Adding membership to the above created domain groups.
    Trace-Execution "Adding group memberships"
    foreach ($domainGroup in $domainGroups) {
        if($isHyperConverged -and $domainGroup.Id -eq "ComputeNodes")
        {
            Trace-Execution "Skipping Group $($domainGroup.Id) as its a HyperConverged topology."
        }
        else
        {
            Trace-Execution "Adding memberships for Group $($domainGroup.Id)"
            foreach ($membership in $domainGroup.Membership.MemberOf) {
                $null = Add-GroupMembership -ADRemoteSession $primaryDomainControllerSession -GroupName $domainGroup.Name -ParentGroupName $membership.GroupName -IsIdempotentRun $IsIdempotentRun
            }
        }
    }

    Trace-Execution "Processing domain users defined in the manifest."
    #creating domain users
    $resetPassword = (Get-RestoreParameters $Parameters).RestoreInProgress
    foreach ($domainUser in $domainUsers) {
        # by default all accounts are enabled
        $accountEnabled = $true
        # disable account if requested by configuration (Disabled="true" attribute)
        if ($domainUser.Disabled -and [Boolean]::Parse($domainUser.Disabled)) {
            $accountEnabled = $false
        }
        $domainUserCredential = $Parameters.GetCredential($domainUser.Credential)
        $null = New-UserAccount -ADRemoteSession $primaryDomainControllerSession -UserAccounts $domainUserCredential -DomainFqdn $DomainFQDN -Password $domainUserCredential.Password -ResetPassword:$resetPassword -Enabled $accountEnabled -IsIdempotentRun $IsIdempotentRun
        foreach ($acl in $domainUser.Acls.Acl) {
            $null = Add-AccountAcls -ADRemoteSession $primaryDomainControllerSession -UserAccount $domainUserCredential.UserName -ObjectName $acl.ObjectName -Permission $acl.Permission -DomainFqdn $domainFqdn
        }
    }

    # adding membership to the above users
    foreach ($domainUser in $domainUsers) {
        $domainUserCredential = $Parameters.GetCredential($domainUser.Credential)
        foreach($membership in $domainUser.Membership.MemberOf)
        {
            $null = Add-UserMembership -ADRemoteSession $primaryDomainControllerSession -ParentGroupName $membership.GroupName -UserIdentityName $domainUserCredential.GetNetworkCredential().UserName -IsIdempotentRun $IsIdempotentRun
        }
    }

    Trace-Execution "Processing service accounts defined in the manifest."
    # creating service accounts
    foreach ($serviceAccount in $serviceAccounts) {
        $null = New-ServiceAccount -ADRemoteSession $primaryDomainControllerSession -ServiceAccounts $serviceAccount -DomainFQDN $DomainFQDN -IsIdempotentRun $IsIdempotentRun
    }

    # adding membership
    foreach ($serviceAccount in $serviceAccounts) {
        foreach($membership in $serviceAccount.Membership.MemberOf)
        {
            $null = Add-ServiceAccountMembership -ServiceAccountIdentityName $serviceAccount.Name `
                                                 -ParentGroupName $membership.GroupName `
                                                 -ADRemoteSession $primaryDomainControllerSession `
                                                 -IsIdempotentRun $IsIdempotentRun
        }
    }

    # add SPNs for service accounts
    foreach ($serviceAccount in $serviceAccounts) {
        foreach ($spn in $serviceAccount.ServicePrincipalNames.ServicePrincipalName)
        {
            $null = Add-ServiceAccountSpn -ServiceAccountIdentityName $serviceAccount.Name `
                                          -ServicePrincipalName $spn.Name `
                                          -ADRemoteSession $primaryDomainControllerSession `
                                          -IsIdempotentRun $IsIdempotentRun
        }
    }
}

# Creates a new local user.
function New-LocalUserWrapper
{
      [CmdletBinding()]
      Param
      (
          [Parameter(Mandatory = $true)]
          [PSCredential]
          $Credential,

          [Parameter(Mandatory = $true)]
          [String[]]
          $ComputerName,

          [Parameter(Mandatory = $true)]
          [String]
          $Username,

          [Parameter(Mandatory = $true)]
          [SecureString]
          $Password
      )

      Invoke-Command -ComputerName $ComputerName -Credential $Credential {

          $userAlreadyExists = Get-LocalUser -Name $Using:Username -ErrorAction SilentlyContinue

          if (-not $userAlreadyExists)
          {
              $null = New-LocalUser -Name $Using:Username -Password $Using:Password
          }
      }
}

# Adds user account to the local administrators group.
function Add-LocalAdministrators
{
    [CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [PSCredential]
        $Credential,

        [Parameter(Mandatory = $true)]
        [String[]]
        $ComputerName,

        [Parameter(Mandatory = $true)]
        [String]
        $Username
    )

    Invoke-Command -ComputerName $ComputerName -Credential $Credential {

        $ADMINISTRATORS_SID = 'S-1-5-32-544'

        $localAdministratorsGroupName = (Get-WMIObject Win32_Group -filter "LocalAccount=True AND SID='$ADMINISTRATORS_SID'").Name

        if($localAdministratorsGroupName)
        {
            [string[]] $currentLocalAdministrators = net localgroup $localAdministratorsGroupName

            if ($currentLocalAdministrators -notcontains $Using:Username)
            {
                $null = net localgroup $localAdministratorsGroupName /add $Using:Username
            }
        }
    }
}

# Gets the current active Domain Controller computer name.
function Get-AvailableADComputerName
{
 [CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [String[]]
        $AllADComputerName,

        [Parameter(Mandatory = $false)]
        [PSCredential]
        $RemoteServiceCredentials = $null
    )
    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop
    $domainVM = hostname

    return  $domainVM

    Trace-ECEScript "Getting the current active Domain Controller computer name." {

        # Proactively look for AD running on local machine (in case of multi-node, this will return the DVM).
        $service = Get-Service -Name "NTDS" -ErrorAction SilentlyContinue
        if($service -and $service.Status -eq "Running")
        {
            $activeADComputer = $env:COMPUTERNAME
        }
        else
        {
            # In case of one Node or post deployment, this will return DC01.
            foreach($adComputer in $AllADComputerName)
            {
                if ($RemoteServiceCredentials)
                {
                    try
                    {
                        $psSession = New-PsSession -Credential $RemoteServiceCredentials -ComputerName $adComputer -ErrorAction SilentlyContinue
                        if($psSession)
                        {
                            $service = Invoke-Command -Session $psSession -ScriptBlock { Get-Service -Name "NTDS" -ErrorAction SilentlyContinue }
                        }
                    }
                    finally
                    {
                        if ($psSession)
                        {
                            Remove-PsSession $psSession
                        }
                    }
                }
                else
                {
                    $service = Invoke-Command -ComputerName $adComputer -ScriptBlock { Get-Service -Name "NTDS" -ErrorAction SilentlyContinue } -ErrorAction SilentlyContinue
                }

                if ($service -and $service.Status -eq "Running")
                {
                    $activeADComputer = $adComputer
                    break
                }
            }
        }
    }

    if($activeADComputer)
    {
        Trace-Execution "Found AD running on $activeADComputer"
        return $activeADComputer
    }
    else
    {
        Trace-Error "No Available AD computer found."
    }
}

function New-ServiceAccount
{
    Param
    (
        [Parameter(Mandatory=$true)]
        [PSObject[]]
        $ServiceAccounts,

        [Parameter(Mandatory=$true)]
        [string]
        $DomainFQDN,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession,

        [Bool]
        $IsIdempotentRun=$false
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    foreach ($serviceAccount in $ServiceAccounts) {
        $serviceAccountName = $serviceAccount.Name
        $vmsGroupName = $serviceAccount.PrincipalsAllowedToRetrieveManagedPassword
        $managedPasswordIntervalInDays = $serviceAccount.ManagedPasswordIntervalInDays
        $ServicePrincipalName = $serviceAccount.ServicePrincipalName

        Trace-Execution "Check AD group $vmsGroupName"
        $adGroup = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:vmsGroupName | Tee-Object -Variable adGroup }

        if ($adGroup) {
            if($IsIdempotentRun)
            {
                Trace-Execution "Skipping: Group '$vmsGroupName' already exists in the AD."
            }
            else
            {
                Trace-Execution "Removing the pre-existing group '$vmsGroupName'."
                $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Remove-ADGroup -Identity $adGroup -Confirm:$false }
            }
        }

        if(-not $IsIdempotentRun)
        {
            Trace-Execution "Adding new AD group '$vmsGroupName'."
            $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADGroup -Name $using:vmsGroupName -DisplayName $using:vmsGroupName -Description $using:vmsGroupName -GroupCategory Security -GroupScope Global }
        }

        Trace-Execution "Check AD service account $serviceAccountName"
        $adUser = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADServiceAccount -Filter {Name -like '*'} | ? Name -eq $using:serviceAccountName | Tee-Object -Variable adUser } -ErrorAction Stop

        if ($adUser) {
            if(-not $IsIdempotentRun)
            {
                Trace-Execution "Removing the pre-existing user '$serviceAccountName'."
                $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Remove-ADServiceAccount -Identity $adUser -Confirm:$false }
                Trace-Execution "Setting service account '$serviceAccountName' with VM group name '$vmsGroupName' and managed password interval of '$managedPasswordIntervalInDays' days."
                $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADServiceAccount -Name $using:serviceAccountName -DNSHostName ($using:serviceAccountName + "." + $using:DomainFQDN) -PrincipalsAllowedToRetrieveManagedPassword $using:vmsGroupName -ManagedPasswordIntervalInDays $using:managedPasswordIntervalInDays -KerberosEncryptionType AES256 }
            }
        }
        else
        {
            Trace-Execution "Setting service account '$serviceAccountName' with VM group name '$vmsGroupName' and managed password interval of '$managedPasswordIntervalInDays' days."
            $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADServiceAccount -Name $using:serviceAccountName -DNSHostName ($using:serviceAccountName + "." + $using:DomainFQDN) -PrincipalsAllowedToRetrieveManagedPassword $using:vmsGroupName -ManagedPasswordIntervalInDays $using:managedPasswordIntervalInDays -KerberosEncryptionType AES256 }
        }

        if($ServicePrincipalName -ne $null)
        {
            Trace-Execution "Adding service account '$serviceAccountName' with ServicePrincipalName '$ServicePrincipalName'."
            $null = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADServiceAccount $using:serviceAccountName | Set-AdServiceAccount -ServicePrincipalNames @{Add="$Using:ServicePrincipalName"} -Confirm:$false }
        }
    }
}

function New-UserGroup
{
    Param
    (
        [Parameter(Mandatory=$true)]
        [PSObject[]]
        $Groups,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession,

        [Bool]
        $IsIdempotentRun=$false
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop
    Trace-Execution "Processing new user groups"
    foreach ($group in $Groups) {
        $groupName = $group.Name
        $groupScope = $group.Scope

        Trace-Execution "Check AD group $groupName"
        $adGroup = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:groupName | Tee-Object -Variable adGroup}

        if ($adGroup) {
            if($IsIdempotentRun)
            {
                Trace-Execution "Skipping: the group '$groupName' already exists in AD."
                continue
            }
            Trace-Execution "Removing the pre-existing group '$groupName'."
            Invoke-Command -Session $ADRemoteSession -ScriptBlock { Remove-ADGroup -Identity $adGroup -Confirm:$false }
        }

        Trace-Execution "Adding new AD group '$groupName'."
        Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADGroup -Name $using:groupName -DisplayName $using:groupName -Description $using:groupName -GroupCategory Security -GroupScope $using:groupScope }
    }
    Trace-Execution "Finished processing new user groups"
}

function Add-GroupMembership
{
    param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $GroupName,

        [Parameter(Mandatory=$true)]
        [string]
        $ParentGroupName,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession,

        [Bool]
        $IsIdempotentRun=$false
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    Trace-Execution "Check AD group $GroupName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $childGroup = Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:GroupName }

    Trace-Execution "Check AD group $ParentGroupName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $parentGroup =  Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName }

    Trace-Execution "Check membership of $GroupName in $ParentGroupName"
    $member = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $using:ParentGroupName | ? Name -EQ $using:GroupName }
    if($IsIdempotentRun -and $member)
    {
        Trace-Execution "Skipping: Group '$GroupName' is already a member of the group '$ParentGroupName'."
        return
    }

    Trace-Execution "Adding group '$GroupName' as a member of the group '$ParentGroupName'."
    Invoke-Command -Session $ADRemoteSession -ScriptBlock {Add-ADGroupMember -Identity $parentGroup -Members $childGroup }
    Trace-Execution "Finished adding group '$GroupName' as a member of the group '$ParentGroupName'."
}

function Add-MachineMembership
{
    param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $MachineName,

        [Parameter(Mandatory=$true)]
        [string]
        $ParentGroupName,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    Trace-Execution "Check AD computer $MachineName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $machineObject = Get-ADComputer -Filter {Name -like '*'} | ? Name -eq $using:MachineName }

    Trace-Execution "Check AD group $ParentGroupName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $groupObject =  Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName }

    Trace-Execution "Check membership of $MachineName in $ParentGroupName"
    $isMember = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $groupObject |? Name -eq $using:MachineName }

    if ($isMember)
    {
        Trace-Execution "Skipping: Machine '$MachineName' is already a member of the group '$ParentGroupName'."
        return
    }

    Trace-Execution "Adding machine '$MachineName' as a member of the group '$ParentGroupName'."
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { Add-ADGroupMember -Identity $groupObject -Members $machineObject }
    Trace-Execution "Finished adding machine '$MachineName' as a member of the group '$ParentGroupName'."
}

function Add-UserMembership
{
    param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $UserIdentityName,

        [Parameter(Mandatory=$true)]
        [string]
        $ParentGroupName,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession,

        [Bool]
        $IsIdempotentRun=$false
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    Trace-Execution "Check AD user $UserIdentityName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $userIdentity = Get-ADUser -Filter {Name -like '*'} | ? Name -eq $using:UserIdentityName }

    Trace-Execution "Check AD group $ParentGroupName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $parentGroup =  Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName }

    Trace-Execution "Check membership of $UserIdentityName in $ParentGroupName"
    $member = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $using:ParentGroupName | ? Name -EQ $using:UserIdentityName }
    if($IsIdempotentRun -and $member)
    {
        Trace-Execution "Skipping: User '$UserIdentityName' is already a member of the group '$ParentGroupName'."
        return
    }

    Trace-Execution "Adding user '$UserIdentityName' as a member of the group '$ParentGroupName'."
    Invoke-Command -Session $ADRemoteSession -ScriptBlock {  Add-ADGroupMember -Identity $parentGroup -Members $userIdentity }
    Trace-Execution "Finished adding user '$UserIdentityName' as a member of the group '$ParentGroupName'."
}

function Remove-UserMembership
{
    param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $UserIdentityName,

        [Parameter(Mandatory=$true)]
        [string]
        $ParentGroupName,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession,

        [Bool]
        $IsIdempotentRun=$false
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    Trace-Execution "Check AD user $UserIdentityName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $userIdentity = Get-ADUser -Filter {Name -like '*'} | ? Name -eq $using:UserIdentityName }

    Trace-Execution "Check AD group $ParentGroupName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $parentGroup =  Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName }

    Trace-Execution "Check membership of $UserIdentityName in $ParentGroupName"
    $member = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $using:ParentGroupName | ? Name -EQ $using:UserIdentityName }
    if($member -or $IsIdempotentRun)
    {
        Trace-Execution "Removing user '$UserIdentityName' as a member of the group '$ParentGroupName'."
        Invoke-Command -Session $ADRemoteSession -ScriptBlock {  Remove-ADGroupMember -Identity $parentGroup -Members $userIdentity -Confirm:$false }
    }
    else
    {
        Trace-Execution "Skipping: User '$UserIdentityName' is already removed from the group '$ParentGroupName'."
    }

    Trace-Execution "Finished removing user '$UserIdentityName' from the group '$ParentGroupName'."
    return
}

function Add-ServiceAccountMembership
{
    param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $ServiceAccountIdentityName,

        [Parameter(Mandatory=$true)]
        [string]
        $ParentGroupName,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession,

        [Bool]
        $IsIdempotentRun=$false
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    Trace-Execution "Check AD service account $ServiceAccountIdentityName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $userIdentity = Get-ADServiceAccount -Filter {Name -like '*'} | ? Name -eq $using:ServiceAccountIdentityName }

    Trace-Execution "Check AD group $ParentGroupName"
    Invoke-Command -Session $ADRemoteSession -ScriptBlock { $parentGroup =  Get-ADGroup -Filter {Name -like '*'} | ? Name -eq $using:ParentGroupName }

    Trace-Execution "Check membership of $ServiceAccountIdentityName in $ParentGroupName"
    $member = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADGroupMember -Identity $using:ParentGroupName | ? Name -EQ $using:ServiceAccountIdentityName }
    if($IsIdempotentRun -and $member)
    {
        Trace-Execution "Skipping: User '$ServiceAccountIdentityName' is already a member of the group '$ParentGroupName'."
        return
    }

    Trace-Execution "Adding user '$ServiceAccountIdentityName' as a member of the group '$ParentGroupName'."
    Invoke-Command -Session $ADRemoteSession -ScriptBlock {  Add-ADGroupMember -Identity $parentGroup -Members $userIdentity }
    Trace-Execution "Finished adding user '$ServiceAccountIdentityName' as a member of the group '$ParentGroupName'."
}

function Add-ServiceAccountSpn
{
    param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $ServiceAccountIdentityName,

        [Parameter(Mandatory=$true)]
        [string]
        $ServicePrincipalName,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession,

        [Bool]
        $IsIdempotentRun=$false
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    $existingSpnNames = Invoke-Command -Session $ADRemoteSession -ScriptBlock {
        $userIdentity = Get-ADServiceAccount -Filter {Name -like '*'} -Properties ServicePrincipalNames | ? Name -eq $using:ServiceAccountIdentityName
        $userIdentity.ServicePrincipalNames
        }

    if ($IsIdempotentRun -and $ServicePrincipalName -in $existingSpnNames)
    {
        Trace-Execution "Skipping: SPN '$ServicePrincipalName' is already created for account '$ServiceAccountIdentityName'."
        return
    }

    Trace-Execution "Adding SPN '$ServicePrincipalName' to user account '$ServiceAccountIdentityName'."

    Invoke-Command -Session $ADRemoteSession -ScriptBlock { Set-ADServiceAccount -Identity $userIdentity -ServicePrincipalNames @{Add=$using:ServicePrincipalName}}
}

function New-UserAccount
{
    Param (

        [Parameter(Mandatory=$true)]
        [PSCredential[]]
        $UserAccounts,

        [Parameter(Mandatory=$true)]
        [string]
        $DomainFqdn,

        [Parameter(Mandatory=$true)]
        [SecureString]
        $Password,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession,

        [Switch]
        $ResetPassword,

        [Bool]
        $Enabled=$true,

        [Bool]
        $IsIdempotentRun=$false
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    foreach ($account in $UserAccounts) {

        $userName = $account.GetNetworkCredential().UserName

        Trace-Execution "Check AD user $userName"
        $adUser = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Get-ADUser -Filter {Name -like '*'} | ? Name -eq $using:userName | Tee-Object -Variable adUser } -ErrorAction Stop

        if ($adUser) {
            if($IsIdempotentRun)
            {
                # Enable/disable account if detected a change in state
                if ($adUser.Enabled -ne $Enabled)
                {
                    $stateName = if ($Enabled) { "Enabling" } else { "Disabling" }
                    Trace-Execution "$stateName the user '$userName' in AD."
                    Invoke-Command -Session $ADRemoteSession -ScriptBlock { Set-ADUser -Identity $adUser -Enabled $using:Enabled -Confirm:$false }
                }

                if ($ResetPassword)
                {
                    Trace-Execution "Resetting the password for the user '$userName', which already exists in AD."
                    Invoke-Command -Session $ADRemoteSession -ScriptBlock { Set-ADAccountPassword -Identity $adUser -NewPassword $using:Password -Confirm:$false }
                }
                else
                {
                    Trace-Execution "Skipping: the user '$userName' already exists in AD."
                }
                continue
            }
            Trace-Execution "Removing the pre-existing user '$userName'."
            Invoke-Command -Session $ADRemoteSession -ScriptBlock { Remove-ADUser -Identity $adUser -Confirm:$false }
        }

        #BUG 9716605: We should be taking expiration period from customer config file.
        Trace-Execution "Adding new AD user '$userName'."
        Invoke-Command -Session $ADRemoteSession -ScriptBlock { New-ADUser -Name $using:userName -GivenName $using:userName -AccountPassword $using:Password -UserPrincipalName "$using:userName@$using:DomainFqdn" -Enabled $using:Enabled -PasswordNeverExpires $true -KerberosEncryptionType AES256 }
        Trace-Execution "Finished adding new AD user '$userName'."
    }
}

function Add-AccountAcls
{
    Param (
        [Parameter(Mandatory=$true)]
        [string]
        $UserAccount,

        [Parameter(Mandatory=$true)]
        [string]
        $ObjectName,

        [Parameter(Mandatory=$true)]
        [string]
        $Permission,

        [Parameter(Mandatory=$true)]
        [string]
        $DomainFqdn,

        [Parameter(Mandatory=$true)]
        [System.Management.Automation.Runspaces.PSSession]
        $ADRemoteSession
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    Trace-Execution "Setting ACL permission '$Permission' for account '$UserAccount' and object '$ObjectName'."

    try {
        $domainNames = $DomainFqdn.Split('.')
        foreach ($domainName in $domainNames) {
            $objectName += ",DC=$domainName"
        }

        $dsaclsCmdline = "DSACLS `"$objectName`" /G `"$DomainFqdn\$UserAccount`:$Permission`""
        Trace-Execution "Executing the following command on '$($ADRemoteSession.ComputerName)': '$dsaclsCmdline'."

        [string] $dsaclsOutput = Invoke-Command -Session $ADRemoteSession -ScriptBlock { Invoke-Expression "$using:dsaclsCmdline" }
    }
    catch {
        Trace-Error "Could not set ACL permissions for account '$UserAccount'. Reason: $_"
    }
}

# Virtual Network Common
function Test-NetworkMap {
    param (
        [Parameter(Mandatory=$true)]
        $NetworkMap,

        [Parameter(Mandatory=$false)]
        [UInt32] $RetryCount = 0,

        [Parameter(Mandatory=$false)]
        [UInt32] $RetryIntervalInSeconds = 60,

        [Parameter(Mandatory=$false)]
        [string[]] $NodeNames
    )

    Trace-ECEScript $LocalizedNetworkData.CheckNICConnectivity {

        $nodesToCheck = $NetworkMap.Keys
        if ($NodeNames) {
            $nodesToCheck = $nodesToCheck | Where-Object {$_ -in $NodeNames}
        }

        $retry = 0
        do
        {
            try
            {
                # Test nic connectivity for each machine
                foreach ($nodeName in ($nodesToCheck | Sort-Object))
                {
                    $node = $NetworkMap.$nodeName
                    $hasPingableConnections = $false

                    foreach ($nicId in ($node.Keys | Sort-Object))
                    {
                        $nic = $NetworkMap.$nodeName.$nicId
                        try
                        {
                            # This cast will fail if the IP address isn't well-formed
                            $ipAddress = [ipaddress]$nic.IPv4Address.Split('/')[0]
                            $canPing = Test-IPConnection $ipAddress.IPAddressToString
                        }
                        catch
                        {
                            $canPing = $false
                        }

                        if ($canPing) {
                            $hasPingableConnections = $true
                        }

                        $nic | Add-Member -MemberType NoteProperty -Name CanPing -Value $canPing

                        Trace-Execution "$(if ($canPing) {'+'} else {'-'}) $nodeName | $($nic.Name)"
                    }

                    if (-not $hasPingableConnections) {
                        throw ($LocalizedNetworkData.NoPingableConnections -f $nodeName)
                    }
                }
                break
            }
            catch
            {
                if($retry -eq $RetryCount) {
                    throw
                }
                Trace-Execution "Test-NetworkMap failed. Will retry in $RetryIntervalInSeconds seconds. Retry attempted: $retry."
                Start-Sleep -Seconds $RetryIntervalInSeconds
            }
            $retry++
        } while ($retry -le $RetryCount)
    }
}

<#
.Synopsis
     Function to clear a potentially remote DNS server cache from any context.
.Parameter DNSServerName
     The DNS server whose cache should be cleared defaulting to the local machine.
.Parameter DNSServerCredential
    Credentials to use when resetting the cache, only used when the server name is also specified.
.Example
    Clear-RemoteDNSServerCache
 
    This will clear the local DNS server's cache using the current credentials
#>
function Clear-RemoteDNSServerCache {
    param (
        [Parameter(Mandatory=$false)]
        [string]
        $DNSServerName = $null,

        [Parameter(Mandatory=$false)]
        [PSCredential]
        $DNSServerCredential = $null
        )

    Trace-Execution "Clear DNS server cache."

    try
    {
        if ($DNSServerName)
        {
            Trace-Execution "Remote DNS server $DNSServerName used."
            $dnsSession = New-PsSession -ComputerName $DNSServerName -Credential $DNSServerCredential
        }
        else
        {
            $dnsSession = New-PsSession
        }

        $null = Invoke-Command -Session $dnsSession -ScriptBlock { Clear-DnsServerCache -Force -ErrorAction SilentlyContinue }
    }
    finally
    {
        $dnsSession | Remove-PsSession
    }

    Trace-Execution "DNS server cache cleared."
}

function Get-IsVirtualNetworkAlreadyConfigured {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory=$true)]
        $NetworkMap,

        [Parameter(Mandatory=$true)]
        $IsOneNode
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    # Check that all non-Deployment vNICs can be pinged.
    $notPingableAdapters = $NetworkMap.Values.Values | ? CanPing -ne $true

    if (-not $notPingableAdapters)
    {
        Trace-Execution $LocalizedNetworkData.NetworkingAlreadyConfigured
        return $true
    }
 
    return $false
}

# BUGBUG: This function may not needed any more in ASZ. Leave it here to not break existing code. Need refactoring.
function Get-NetworkMap {

    [CmdletBinding()]
    param (
        [Parameter(Mandatory=$true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        $PhysicalMachinesRole
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    $nodeNames = $PhysicalMachinesRole.Nodes.Node | % Name

    Trace-ECESCript "Get Network Map for nodes $nodeNames" {

        # If multiple nodes are being added at the same time, only test those that are being deployed by the current action plan instance
        $executionRoleName = $Parameters.Context.ExecutionContext.Roles.Role.RoleName
        $hostsList = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name

        if ($executionRoleName -ieq "Cluster")
        {
            $hostsList = $null
            $hostsList = [array]($Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.PhysicalNodes.PhysicalNode.Name)
        }

        $networkmap = @{}
        $allHostNodes = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters

        foreach ($node in $PhysicalMachinesRole.Nodes.Node)
        {
            if (-not($node.OperationStatus -eq "RequiresRemediation") -and (-not($node.OperationStatus -eq "Adding") -or ($hostsList -contains $node.Name)))
            {
                # there would be only 1 IP on each host that may need checking.
                $nodeNics = @{}
                
                $nicIPv4 = $allHostNodes[$node.Name]
                $nicName = $node.Name

                if (-not (Test-NetworkIPv4Address -IPv4Address $nicIPv4))
                {
                    Trace-Execution "[Get-NetworkMap]: IPv4 Address for node $($node.Name) $nicIPv4 is not with expected format, skip."
                    continue
                }

                $nicData = New-Object PSObject -Property @{
                    Name = $nicName
                    IPv4Address = $nicIPv4
                }

                $nodeNics.$nicName = $nicData
                $networkMap.($node.Name) = $nodeNics
            }
        }
    }

    return $networkMap
}

function Wait-VirtualNetwork {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory=$true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        $PhysicalMachinesRole,

        [Boolean]
        $IsIdempotentRun = $true,

        [Parameter(Mandatory=$false)]
        [string] $UpdateNodeName
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    Trace-ECEScript "Waiting for Virtual Network to be configured on the host." {
        $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
        $securityInfo = $cloudRole.PublicInfo.SecurityInfo
        $localAdminUser = $securityInfo.LocalUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.BuiltInAdminAccountID
        $credential = $Parameters.GetCredential($localAdminUser.Credential)
        Set-MacAndIPAddress -Parameters $Parameters -PhysicalMachinesRole $PhysicalMachinesRole -IPv4AddressSource 'Management'

        # in the case of Bare Metal the hosts, it is possible that the DSC hasn't completed the configuration for the hosts at this point,
        # so add some retry logic to test the network connectivity
        $networkMap = Get-NetworkMap -Parameters $Parameters -PhysicalMachinesRole $PhysicalMachinesRole
        if ($UpdateNodeName) {
            Test-NetworkMap -NetworkMap $networkMap -RetryCount 10 -NodeNames @($UpdateNodeName)
        }
        else {
            Test-NetworkMap -NetworkMap $networkMap -RetryCount 10
        }

        $nodes = $PhysicalMachinesRole.Nodes.Node
        $nodeNames = $nodes | ForEach-Object Name
        $jobs = @()

        $isOneNode = $false
        if ($UpdateNodeName) {
            # For update, only one host is service at a time and this should not get confused with One Node MAS environment
            $nodes = $nodes | Where-Object Name -eq $UpdateNodeName
        }
        elseif ($nodeNames.Count -eq 1 ) {
            $isOneNode = $true
            # MASD One Node will always be run locally
            $nodeNames = 'LocalHost'
        }

        if ($IsIdempotentRun) {
            if (Get-IsVirtualNetworkAlreadyConfigured -NetworkMap $networkMap -IsOneNode $isOneNode) {
                return
            }
        }

        $allHostNodes = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters
        if ($env:OSImageType -ne "ISO") {
            Trace-ECEScript "Start PowerShell jobs to retrieve and set the DSC state on all nodes $nodeNames." {
                foreach ($node in $nodes) {
                    # Collect information about the state of the recently deployed physical host.
                    $managementIpAddress = $allHostNodes[$node.Name]
                    $nodeName = $node.Name
                    Trace-Execution "Sending DSC query to $managementIpAddress for $nodeName"
                    $jobs += Start-Job -ScriptBlock {
                        $retVal = $false
                        $logText = "DSC report for node $(($using:Node).Name)`r`n"

                        # Wait as much as ten minutes for the exiting DSC configuration to complete. This is just a best effort.
                        for ($i = 0; $i -lt 40; $i++) {
                            $cimSession = $null
                            $result = $null
                            try {
                                # This call may fail if it can't connect to the remote host when remote host is configuring vmswitch and host vnic.
                                $cimSession = New-CimSession -ComputerName $Using:nodeName -Credential $using:credential -ErrorAction SilentlyContinue
                                if ($null -eq $cimSession) {
                                    $cimSession = New-CimSession -ComputerName $using:managementIpAddress -Credential $using:credential -ErrorAction Stop
                                }
                                # This call will throw exception if there is currently a DSC config running on the host.
                                # we use it as a way as an indication.
                                $logText += "Calling Get-DscConfigurationStatus at attempt #$i. `r`n"
                                $result = Get-DscConfigurationStatus -CimSession $cimSession -ErrorAction Stop
                                break
                            }
                            catch {
                                Start-Sleep -Seconds 15
                                $logText += "Get-DscConfigurationStatus failed with exception ($($_.Exception.Message)) at attempt #$i `r`n"
                            }
                            finally {
                                if ($cimSession) {
                                    Remove-CimSession $cimSession
                                }
                            }
                        }

                        $logText += "Trying to make sure host DSC configuration is completed. `r`n"
                        for ($i = 0; $i -lt 5; $i++) {
                            $cimSession = $null
                            try {
                                $cimSession = New-CimSession -ComputerName $Using:nodeName -Credential $using:credential -ErrorAction SilentlyContinue
                                if ($null -eq $cimSession) {
                                    $cimSession = New-CimSession -ComputerName $using:managementIpAddress -Credential $using:credential -ErrorAction Stop
                                }
                                $logText += "Calling Start-DscConfiguration at attempt #$i `r`n"
                                Start-DscConfiguration -UseExisting -Wait -CimSession $cimSession -Verbose -Force -ErrorAction Stop
                                $logText += "Calling Get-DscConfigurationStatus at attempt #$i. `r`n"
                                $result = Get-DscConfigurationStatus -CimSession $cimSession -ErrorAction Stop
                                $logText += "Calling Get-DscConfiguration at attempt #$i. `r`n"
                                $logText += (Get-DscConfiguration -CimSession $cimSession -Verbose -ErrorAction Stop | out-string)
                            }
                            catch {
                                # While DSC configuration on the host is running, it may cause the network connectivity to be disrupted.
                                # If that disruption last too long, say more than 3 or 4 minutes, the above calls may fail with exception.
                                # So catch the exception here but do nothing.
                                $logText += "Attempt #$i : failed with exception ($($_.Exception.Message)). `r`n"
                            }
                            finally {
                                if ($result.status -eq 'Success') {
                                    $retVal = $true
                                }

                                if ($cimSession) {
                                    Remove-CimSession $cimSession
                                }
                            }
                            if ($retVal) {
                                break
                            }
                        }

                        $retObj = @{RetVal = $retVal; LogText = $logText; HostIp = $using:managementIpAddress}
                        $retObj
                    }
                }
            }

            # Wait for all the nodes to either converge or fail to converge. If they failed to
            # converge, or if we failed to send the WSManAction to them, then use the more stateful
            # networking methods to figure out why, later.
            Trace-ECEScript "Waiting for DSC query to complete" {
                $results = Receive-Job -Wait -Job $jobs
                $results | ForEach-Object {
                    Trace-Execution $_.LogText
                    if ($_.RetVal -ne $true) {
                        Trace-Error "DSC failed to converge on a physical host $($_.HostIp)."
                    }
                }
            }

            $jobs | Remove-Job -ErrorAction SilentlyContinue
        }
        else {
            Trace-Execution "Skipping wait for DSC config as this is an ISO-based deployment."
        }
    }
}

function Get-DomainIPMapping
{
    param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $domainIPMapping = @{}
    $deploymentMachineRoleNode = $Parameters.Roles["DeploymentMachine"].PublicConfiguration.Nodes.Node

    if($deploymentMachineRoleNode -ne $null)
    {
        $domainIPMapping.Add($deploymentMachineRoleNode.Name, $deploymentMachineRoleNode.IPv4Address)
    }
    
    return $domainIPMapping
}

# Get the node names from the ExecutionContext in the $Parameters variable
# if available; otherwise, return $null
function Get-ExecutionContextNodeName
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $false)]
        [switch]
        $EnsureSingle
    )

    $nodeName = $null
    $executionRoleName = $Parameters.Context.ExecutionContext.Roles.Role.RoleName
    $nodeName = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name

    if ($null -ne $executionRoleName -and $null -eq $nodeName)
    {
        # If node names are not provided, get all nodes associated to the $executionRoleName
        $nodeName = ($Parameters.Roles[$executionRoleName].PublicConfiguration.Nodes.Node.Name) | Select-Object -Unique
    }

    Trace-Execution "Received ExecutionContext with role name $executionRoleName and node(s): $($nodeName -join ', ')"

    if ($EnsureSingle -and (!$nodeName -or ($nodeName.Count -ne 1)))
    {
        throw "Expected single node in ExecutionContext but found node(s): $($nodeName -join ', ')"
    }

    return $nodeName
}

# Get the cluster name from the ExecutionContext in the $Parameters variable
# if available; otherwise, return $null
function Get-ExecutionContextClusterName
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $executionRoleName = $Parameters.Context.ExecutionContext.Roles.Role.RoleName
    $clusterName = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.ClusterName
    Trace-Execution "Received ExecutionContext with role name $executionRoleName and clusterName : $clusterName"

    return $clusterName
}

# Gets the names of the roles specified in the execution context.
function Get-ExecutionContextRoleName
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    # ExecutionContext can specify RolePath and/or RoleName attributes. Since we want just the names
    # we truncate the role paths if they are the only attribute provided, and return a list of the unique role names.
    $executionRoleName = @( $Parameters.Context.ExecutionContext.Roles.Role.RoleName )
    $executionRolePath = @( $Parameters.Context.ExecutionContext.Roles.Role.RolePath )

    $executionRoleName += $executionRolePath | Split-Path -Leaf
    return ($executionRoleName | Select-Object -Unique)
}

# Determines whether the environment is one-node from the interface parameters.
function IsOneNode
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    return @($Parameters.Roles["Storage"].PublicConfiguration.Nodes.Node).Count -eq 1
}

# Determines whether the environment is virtual from the interface parameters.
function IsVirtualAzureStack
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $oemModel = $Parameters.Roles["OEM"].PublicConfiguration.PublicInfo.UpdatePackageManifest.UpdateInfo.Model
    return $oemModel -eq "Hyper-V"
}

# This function checks whether the deployment is running on Virtual Machine by system prams
function IsOnVirtualMachine
{
    try
    {
        # is on VM
        $computerSystem = Get-WmiObject win32_computersystem
        $isOnVM = (('Microsoft Corporation' -eq $computerSystem.Manufacturer) -and ('Virtual Machine' -eq  $computerSystem.Model))
    }
    catch
    {
        Trace-Error "Error while checking system info for Virtual Machine deployment"
        return $false
    }

    return $isOnVM
}

# This function starts each service in serial if not running, and waits for each service to be Running
function Assert-Service {

    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [System.String[]]
        $Name,

        [Parameter(Mandatory = $true)]
        [System.String]
        $ComputerName,

        [Parameter(Mandatory = $true)]
        [PSCredential]
        $Credential,

        [Parameter(Mandatory = $false)]
        [switch]
        $WaitOnly,

        [Parameter(Mandatory = $false)]
        [ScriptBlock]
        $WaitScript
    )

    $ErrorActionPreference = 'Stop'
    $timeoutInSeconds = 120

    Trace-Execution "Ensure services '$Name' on $ComputerName machine are running."
    $psSession = New-PSSession -ComputerName $ComputerName -Credential $Credential
    $services = Invoke-Command $psSession {Get-Service -Name $using:Name -ErrorAction Stop}

    # Normally all services should start automatically when machine is turned on
    foreach ($serviceController in $services)
    {
       $serviceName = $serviceController.Name

       if ($serviceController.Status -ne 'Running')
       {
          Trace-Execution "Service '$serviceName' is not running."
          if (-not $WaitOnly)
          {
              Trace-Execution "Starting the service '$serviceName'."
              Invoke-Command $psSession { Start-Service -Name $using:serviceName -ErrorAction Stop }
          }

          $waitServiceScript = { $service = Invoke-Command $psSession {Get-Service -Name $using:serviceName -ErrorAction Stop}; ($service.Status -eq 'Running') }

          if (Wait-Result -ValidationScript $waitServiceScript -TimeOut ($timeoutInSeconds) -Interval 10)
          {
              Trace-Execution "Service '$serviceName' has successfully started."
          }
          else
          {
              Trace-Error ($LocalizedStrings.ServiceDidNotStartup -f $serviceName,$ComputerName,$timeoutInSeconds)
          }
       }
       else
       {
          Trace-Execution "Service '$serviceName' is already running."
       }

       if($waitScript)
       {
            Trace-Execution "Running wait script to ensure service can receive requests."

            if (Wait-Result -ValidationScript $waitScript -TimeOut ($timeoutInSeconds) -Interval 10)
            {
                Trace-Execution "Wait script completed successfully."
            }
            else
            {
                Trace-Error ($LocalizedStrings.ServiceDidNotStartup -f $serviceName, $ComputerName, $timeoutInSeconds)
            }
        }
    }
}

# Checks the RunInformation for UpdateVersion. Returns the value if found else returns null
function Get-InProgressUpdateVersion
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $version = $null
    Trace-Execution "Getting in progress Update Version."
    # In case of update, run time information will be populated
    if($Parameters.RunInformation -ne $null)
    {
        $runtimeParameters = $Parameters.RunInformation['RuntimeParameter']

        if( $runtimeParameters -ne $null -and $runtimeParameters.ContainsKey('UpdateVersion') )
        {
            $version = $runtimeParameters["UpdateVersion"]
            Trace-Execution "Version of the update being applied : $version"
            return $version
        }
    }

    # If it is a one node or bootstrap VM return null as an update will not run on it.
    $buildLocally = $Parameters.Context.ExecutionContext.BuildVhdLocally -and [bool]::Parse($Parameters.Context.ExecutionContext.BuildVhdLocally)
    $RoleName = $Parameters.Configuration.Role.Id
    $isOneNode = @($Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node).Count -eq 1
    if(($isOneNode -eq $true) -or ($buildLocally -and ($RoleName -eq "Domain")))
    {
        return $version
    }

    # If the version is set to null, check if share has update version info file.
    # This file only exists while a host is being updated.
    if($version -eq $null)
    {
        $bareMetalRole = $Parameters.Roles["BareMetal"].PublicConfiguration
        $storageClusterName = Get-ManagementClusterName $Parameters
        if($bareMetalRole.PublicInfo.UpdateInformation -ne $null)
        {
            $updateInfoFolderPath = Get-SharePath $Parameters $bareMetalRole.PublicInfo.UpdateInformation.UpdateVersionFile.FolderName $storageClusterName
            $updateInfoFileName = $bareMetalRole.PublicInfo.UpdateInformation.UpdateVersionFile.FileName
            $filePath = Join-Path -Path $updateInfoFolderPath -ChildPath $updateInfoFileName
            Trace-Execution "Check if version file $filePath exists"
            if(Test-Path -Path $filePath)
            {
                Trace-Execution "The version file is present. Extracting version number and override status from it"
                $versionFileContent = [xml] ( Get-Content -Path $filePath )
                if($versionFileContent)
                {
                    $versionValue = $versionFileContent.InProgressUpdateVersion.Version
                    $useThisVersion = $versionFileContent.InProgressUpdateVersion.UseThisVersion
                    if($useThisVersion -eq "True")
                    {
                        $version = $versionValue
                    }
                    else
                    {
                        Trace-Error "The version file exists but is not being used as UseThisVersion flag is set to False. This flag should be set to True if FRU is being used to recover from an update failure"
                    }
                }
                else
                {
                    Trace-Error "Version file found at $filePath but has invalid content"
                }
            }
        }
    }

    return $version
}

# Gets the file share directory containing expanded contents of a currently in-progress update package.
function Get-InProgressUpdatePackagePath
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )
    $runtimeParameters = $Parameters.RunInformation['RuntimeParameter']
    $packagePath = $runtimeParameters["UpdatePackagePath"]
    if (-not $packagePath)
    {
        Trace-Execution "The UpdatePackagePath runtime parameter is not present."
        return $null
    }

    Trace-Execution "Upgrade package @: [$packagePath]"

    # The package path is expected to be a full path to an update package file (e.g. a self-
    # extracting .exe, or a metadata.xml file). The path where the contents are extracted is
    # the directory containing this file.
    return (Split-Path $packagePath)
}

# Gets the location of the folder containing nugets in the currently in-progress update package.
function Get-InProgressUpdateNugetStore
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $updatePackagePath = Get-InProgressUpdatePackagePath -Parameters $Parameters
    if ($updatePackagePath)
    {
        $urpRole = $Parameters.Roles["URP"].PublicConfiguration
        $nugetSourceFolderName = $urpRole.PublicInfo.UpdatePackagePaths.NugetFolder.Name

        $nugetStorePath = Join-Path $updatePackagePath $nugetSourceFolderName
        if (Test-Path $nugetStorePath)
        {
            return $nugetStorePath
        }
    }
    else
    {
        Trace-Execution "No update is currently in progress."
    }

    return $null
}

<#
.SYNOPSIS
Expands the deployment artifacts to the destination location.
 
.DESCRIPTION
Expands all deployment artifacts into the destination root path.
 
.EXAMPLE
Expand-DeploymentArtifacts -DeploymentContentNode $Parameters.Configuration.Role.PrivateInfo.DeploymentContent -DestinationRootPath "C:"
 
.PARAMETER DeploymentContentNode
The xml node containing the deployment content to deliver to the destination.
 
.PARAMETER DestinationRootPath
The root path under which to deliver the nuget content.
#>
function Expand-DeploymentArtifacts
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory=$true)]
        [ValidateNotNull()]
        [System.Xml.XmlElement]
        $DeploymentContentNode,

        [Parameter(Mandatory=$true)]
        [string]
        $DestinationRootPath,

        [Parameter(Mandatory=$false)]
        [string]
        $NugetStorePath = "$env:SystemDrive\CloudDeployment\NuGetStore"
    )

    $nugetDeploymentContentItems = $DeploymentContentNode.NugetArtifact

    $installRolePackages = $false
    $rolePackageDestination = Join-Path $DestinationRootPath 'NugetStore'
    foreach( $deployContent in $nugetDeploymentContentItems )
    {
        Write-Verbose -Verbose "Copying content from nuget $($deployContent.Name) into $DestinationRootPath"
        $destination = Join-Path $DestinationRootPath $deployContent.DestinationPath

        $sourceIsFile = [bool]$deployContent.SourceIsFile
        $isNugetInstall = [bool]$deployContent.NugetInstall

        if($isNugetInstall)
        {
            $installRolePackages = $true
        }

        Write-Verbose -Verbose "$($deployContent.SourcePath) will be copied to $destination"
        Write-Verbose -Verbose "IsNugetInstall : $isNugetInstall"
        Expand-NugetContent -NugetName $deployContent.Name -SourcePath $deployContent.SourcePath -DestinationPath $destination -SourceIsFile:$sourceIsFile -Verbose:$VerbosePreference -NugetStorePath $NugetStorePath -IsNugetInstall:$isNugetInstall
        Write-Verbose -Verbose "Nuget $($deployContent.Name) with source [$($deployContent.SourcePath)] copied to $destination"
    }

    if($installRolePackages)
    {
        Write-Verbose -Verbose "Install Role Nuget packages to NugetStore"
        Install-RoleNugetPackages -NugetStorePath $NugetStorePath -Destination $rolePackageDestination
    }
}

function Expand-UpdateContent
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory=$true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory=$false)]
        [string[]]
        $ExecutionRoleName,

        [Parameter(Mandatory=$false)]
        [switch]
        $DSCOnly,

        [Parameter(Mandatory=$false)]
        [switch]
        $EngineUpdateOnly,

        [Parameter(Mandatory=$false)]
        [string[]]
        $NodeName
    )

    $domainCredential = Get-DomainCredential -Parameters $Parameters

    $currentContext = $Parameters.Context.ExecutionContext
    if(-not $ExecutionRoleName)
    {
        $ExecutionRoleName = $currentContext.Roles.Role.RoleName
    }

    $vmRole = $Parameters.Roles["VirtualMachines"].PublicConfiguration

    $localExecutionHost = $env:COMPUTERNAME
    $clusterName = Get-ManagementClusterName $Parameters
    $allManagementHosts = Get-ManagementClusterNodes $Parameters

    $uncLibraryShareNugetStorePath = Get-SharePath $Parameters $vmRole.PublicInfo.LibraryShareNugetStoreFolder.Path $clusterName

    # During AsZ deployment expand nugets on all the nodes part of cluster
    # Cluster path at this time doesn't exist so use the absolute path
    if (!(Test-Path -Path $uncLibraryShareNugetStorePath))
    {
        Trace-Execution "AsZ deployment scenario so use absolute path on SeedNode"
        $uncLibraryShareNugetStorePath = "$env:SystemDrive\CloudDeployment\NuGetStore"
    }

    $localLibraryShareNugetStorePath = $null
    # Bug 15111454: Waiting for fix it should not block deployment
    # if(Test-LocalHostHasClusterStoragePath -Parameters $Parameters)
    # {
    # Trace-Execution "Getting local CSV path for $uncLibraryShareNugetStorePath"
    # $localLibraryShareNugetStorePath = Get-LocalCsvPathFromSharePath -Parameters $Parameters -UNCSharePath $uncLibraryShareNugetStorePath
    # }

    Trace-Execution "uncLibraryShareNugetStorePath: $uncLibraryShareNugetStorePath"
    Trace-Execution "localLibraryShareNugetStorePath: $localLibraryShareNugetStorePath"

    $systemDrive = $env:SystemDrive -Replace (':','$')
    $healthyNodes = $Parameters.Roles["VirtualMachines"].PublicConfiguration.Nodes.Node | where { $_.ProvisioningStatus -eq "Complete" } | % Name
    $healthyNodes += $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node | where { $_.ProvisioningStatus -eq "Complete" } | % Name

    foreach ($roleName in $ExecutionRoleName)
    {
        Trace-Execution "Expand-UpdateContent adding update contents for role: $roleName"
        $roleConfiguration = $Parameters.Roles[$roleName].PublicConfiguration
        if ($NodeName)
        {
            $nodes = $NodeName
        }
        else
        {
            $nodes = $roleConfiguration.Nodes.Node.Name |where { $healthyNodes.Contains($_)}
        }

        Trace-Execution "Expand-UpdateContent for Nodes : $($nodes)"

        # Skipping nodes which are provided in runtime parameters
        $nodes = Skip-NodesProvidedInRuntimeParameter -Parameters $Parameters -Nodes $nodes

        Trace-Execution "Expand-UpdateContent for Nodes (after filtering) : $($nodes)"

        # Get update content for the role
        $nugetUpdateContentItems = $roleConfiguration.PublicInfo.DeploymentContent.NugetArtifact | where {$_.Update -eq "True" -or $_.EngineUpdate -eq "True"}

        # Copy the update content
        if ($nugetUpdateContentItems)
        {
            foreach( $updateContent in $nugetUpdateContentItems )
            {
                if($DSCOnly -and ($updateContent.DSCContent -ne "True"))
                {
                    Trace-Execution "$($updateContent.Name) is not marked as nuget with DSC content. Skipping update on the target VM(s) "
                    continue
                }
                elseif ($EngineUpdateOnly -and ($updateContent.EngineUpdate -ne "True"))
                {
                    Trace-Execution "$($updateContent.Name) is not marked as nuget with engine content. Skipping update on the target VM(s)."
                    continue
                }

                foreach ($node in $nodes)
                {
                    $libraryShareNugetStorePath = $null
                    if (($node -in $allManagementHosts) -and ($null -ne $localLibraryShareNugetStorePath))
                    {
                        Trace-Execution "For node $node - member the management cluster - will use local nuget store path $localLibraryShareNugetStorePath"
                        $libraryShareNugetStorePath = $localLibraryShareNugetStorePath
                    }
                    else
                    {
                        Trace-Execution "For node $node will use UNC nuget store path $uncLibraryShareNugetStorePath"
                        $libraryShareNugetStorePath = $uncLibraryShareNugetStorePath
                    }

                    Trace-Execution "Start process to expand nuget content:"
                    Trace-Execution "Local execution host : $localExecutionHost"
                    Trace-Execution "Nuget store Path : $libraryShareNugetStorePath"
                    Trace-Execution "Processing node : $node"

                    $DestinationRootPath = "\\$node\$systemDrive\"
                    if ($updateContent.UpdatePath)
                    {
                        $destination = Join-Path $DestinationRootPath $updateContent.UpdatePath
                    }
                    else
                    {
                        $destination = Join-Path $DestinationRootPath $updateContent.DestinationPath
                    }

                    $isNugetInstall = $null
                    [bool]::TryParse($updateContent.NugetInstall, [ref]$isNugetInstall)

                    $retry = 0
                    $maxRetry = 3
                    while ($retry -lt $maxRetry)
                    {
                        $retry++
                        try
                        {
                            Expand-NugetContent -NugetName $updateContent.Name -SourcePath $updateContent.SourcePath -DestinationPath $destination -IsUnc -NugetStorePath $libraryShareNugetStorePath -IsNugetInstall:$isNugetInstall -Credential $domainCredential
                            Write-Verbose -Verbose "$($updateContent.SourcePath) copied to $destination"
                            break
                        }
                        catch
                        {
                            # Repair item for ICM 127772311
                            if($retry -lt $maxRetry -and $roleName -in ('BareMetal', 'ACS') -and "$_" -match 'being used by another process')
                            {
                                Trace-Execution "Caught exception: $_, try apply remediation..."
                                Import-Module -Name "$PSScriptRoot\..\ACS\ACSRoot.psm1" -DisableNameChecking -Force
                                Stop-DanglingAcsDeploymentPSProcess -Parameters $Parameters -NodeToFix $node
                            }
                            elseif($retry -lt $maxRetry -and $roleName -in ('SeedRingServices'))
                            {
                                Trace-Execution "Caught exception: $_, try apply remediation to close PEP session..."
                                Import-Module $PSScriptRoot\..\SeedRingServices\SeedRingServices.psm1 -DisableNameChecking -Force
                                Close-PEPSessions -Parameters $Parameters
                            }
                            else
                            {
                                throw
                            }
                        }
                    }
                }
            }
        }
    }
}

#execute to a script block over a VMGroup using PSDirect instead of WinRM
function Invoke-PSDirectOnVM
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $true)]
        [String[]]$VMNames,

        [Parameter(Mandatory = $true)]
        [Object]$VMCredential,

        [Parameter(Mandatory = $true)]
        [ScriptBlock]$ScriptBlock,

        [Object[]]$ArgumentList = $null
    )
    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    #retrieve the environment info
    $physicalMachineRole = $Parameters.Roles["BareMetal"].PublicConfiguration
    $domainRole = $Parameters.Roles["Domain"].PublicConfiguration
    $allPhysicalMachines = Get-ActiveClusterNodes $Parameters

    # Account info
    $securityInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo

    # Domain Info
    $domainAdminCredential = Get-DomainCredential -Parameters $Parameters

    # for each physical host, iterate over all the infra VMs that are part of it
    $hostTrace = @()
    foreach ($phyHost in $allPhysicalMachines) {
        Trace-Execution "For Host $($phyHost)"

        # test if the target host is reachable
        $output = $false
        $output = Test-WSMan -ComputerName $phyHost -ErrorAction:Ignore
        if (-not $output)
        {
            Write-Host "Host $($phyHost) not responding..."
            continue
        }
        $localHostTrace = Invoke-Command -ComputerName $phyHost -Credential $domainAdminCredential -ScriptBlock {
            $vmTrace = @()
            $thrownExceptions = @()
            # get the infra VMs in this physical node
            $retries = 5
            for($retry = 0; $retry -lt $retries; ++$retry)
            {
                try
                {
                    # Unfiltered calls to Get-VM may fail if a VM is being deleted from that host
                    $localVMs = (Get-VM | ? Name -In $using:VMNames).Name
                    break
                }
                catch
                {
                    if ($retry -eq $retries - 1)
                    {
                        throw
                    }
                }
            }
            if($localVMs) {
                Trace-Execution "Executing on local VMs $($localVMs -join ',')"
                try {
                    # change PS language mode to FULL to allow script block invocation
                    if (!$using:ArgumentList) {
                        $vmTrace += Invoke-Command -VMName $localVMs -Credential $using:VMCredential -ScriptBlock ([ScriptBlock]::Create($("Import-Module OpenUpSession; Set-FullLanguage; `r`n" + $using:ScriptBlock)))
                    }
                    # check if the scriptblock declares Param because special handling is needed
                    elseif ($using:ScriptBlock -match "param[ ]*\(|param[ ]*\r\n")
                    {
                        $sb = {param($ScriptBlock, [Object[]]$ArgumentList) Import-Module OpenUpSession; Set-FullLanguage; [scriptBlock]::Create($ScriptBlock).invoke($ArgumentList)}
                        $vmTrace += Invoke-Command -VMName $localVMs -Credential $using:VMCredential -ScriptBlock $sb -ArgumentList $using:ScriptBlock, $using:ArgumentList
                    }
                    else
                    {
                        $vmTrace += Invoke-Command -VMName $localVMs -Credential $using:VMCredential -ArgumentList $using:ArgumentList -ScriptBlock ([ScriptBlock]::Create($("Import-Module OpenUpSession; Set-FullLanguage; `r`n" + $using:ScriptBlock)))
                    }
                } catch {
                    Write-Host "Local exception: $($_)"
                    $thrownExceptions += $_
                }
            }
            if ($vmTrace)
            {
                Write-Host "VM(s) returned: $vmTrace"
            }
            if ($thrownExceptions.Count -gt 0) {
               throw $thrownExceptions
            }
            return $vmTrace
        }
        if ($localHostTrace)
        {
            Trace-Execution "Local Host Output: $($localHostTrace)"
        }
        $hostTrace += $localHostTrace
    }
    return $hostTrace
}

function ReliableGetService{
    param (
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Parameter(Mandatory = $true)]
        [String] $ServiceName,
        [Parameter(ParameterSetName="Local")]
        [Switch] $Local,
        [Parameter(ParameterSetName="Remote")]
        [String[]]$Hostnames,
        [Parameter(ParameterSetName="Remote")]
        [PSCredential]$Credential,
        [Parameter(ParameterSetName="VMRemote")]
        [String[]]$VMNames,
        [Parameter(ParameterSetName="VMRemote")]
        [PSCredential]$VMCredential,
        [Parameter(ParameterSetName="VMRemote")]
        [CloudEngine.Configurations.EceInterfaceParameters]$Parameters
    )
    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    $sbGetService = [ScriptBlock]::Create("Get-Service -Name $($ServiceName)")
    Trace-Execution "Retrieving service $ServiceName"
    try {
        if ($PSCmdlet.ParameterSetName -eq "Local") {
            $service = Invoke-Command -ScriptBlock $sbGetService
        } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
            $service = Invoke-Command -ComputerName $Hostnames -Credential $Credential -ScriptBlock $sbGetService
        } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
            $service = Invoke-PSDirectOnVM -Parameters $Parameters -VMNames $VMNames -VMCredential $VMCredential -ScriptBlock $sbGetService
        }
    } catch {
        Trace-Error "Service $ServiceName throw $($_)"
        throw
    }

    if ($service) {
        Trace-Execution "For service $($ServiceName), current status: $($service.Status); start type: $($service.StartType)"
    }

    return $service
}

function ReliableSetService{
    param (
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Parameter(Mandatory = $true)]
        [String] $ServiceName,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Parameter(Mandatory = $true)]
        [ValidateSet("Boot","System","Automatic","Manual","Disabled")]
        [String] $StartupType,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [ValidateSet("Running","Stopped","Pause")]
        [String] $Status,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int32] $MaxRetries = 10,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int] $Delay = 10,
        [Parameter(ParameterSetName="Local")]
        [Switch] $Local,
        [Parameter(ParameterSetName="Remote")]
        [String[]]$Hostnames,
        [Parameter(ParameterSetName="Remote")]
        [PSCredential]$Credential,
        [Parameter(ParameterSetName="VMRemote")]
        [String[]]$VMNames,
        [Parameter(ParameterSetName="VMRemote")]
        [PSCredential]$VMCredential,
        [Parameter(ParameterSetName="VMRemote")]
        [CloudEngine.Configurations.EceInterfaceParameters]$Parameters
    )
    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    $iteration = 0;
    do
    {
        Trace-Execution "$StartupType service $ServiceName"
        if ($Status -eq "") {
            $sbActionService = [ScriptBlock]::Create("Set-Service -Name $($ServiceName) -StartupType $($StartupType)")
        } else {
             $sbActionService = [ScriptBlock]::Create("Set-Service -Name $($ServiceName) -StartupType $($StartupType) -Status $($Status)")
        }
        try {
            if ($PSCmdlet.ParameterSetName -eq "Local") {
                Invoke-Command -ScriptBlock $sbActionService
            } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
                Invoke-Command -ComputerName $Hostnames -Credential $Credential -ScriptBlock $sbActionService
            } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
                Invoke-PSDirectOnVM -Parameters $Parameters -VMNames $VMNames -VMCredential $VMCredential -ScriptBlock $sbActionService
            }
        } catch {
            Trace-Error "Trying to set Service $ServiceName to $($StartupType) throw $($_)"
            throw
        }

        # check service status
        if ($PSCmdlet.ParameterSetName -eq "Local") {
            $serviceVerify = ReliableGetService -ServiceName $ServiceName -Local
        } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
            $serviceVerify = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential
        } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
            $serviceVerify = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters
        }

        if ($Status -eq "") {
            if ($serviceVerify.StartType -eq $StartupType) {
                Trace-Execution "Service $($ServiceName) is $($StartupType)"
                break
            }
        } else {
            if (($serviceVerify.StartType -eq $StartupType) -and ($serviceVerify.Status -eq $Status)) {
                Trace-Execution "Service $($ServiceName) is $($StartupType) and $($Status)"
                break
            }
        }
        $iteration++
        if ($iteration -eq $MaxRetries) {
            if ($Status -eq "") {
                Trace-Error "Couldn't change the Type to $StartupType"
            } else {
                Trace-Error "Couldn't change the Type to $StartupType and $Status"
            }
            throw
        }
        Start-Sleep $Delay
        Trace-Execution "Retrying, iteration $iteration"
    } while ($true);
}

function ReliableActionService{
    param (
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Parameter(Mandatory = $true)]
        [String] $ServiceName,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Parameter(Mandatory = $true)]
        [ValidateSet("Start","Stop", "Restart")]
        [String] $Action,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Boolean] $Force,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int32] $MaxRetries = 10,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int] $Delay = 10,
        [Parameter(ParameterSetName="Local")]
        [Switch] $Local,
        [Parameter(ParameterSetName="Remote")]
        [String[]]$Hostnames,
        [Parameter(ParameterSetName="Remote")]
        [PSCredential]$Credential,
        [Parameter(ParameterSetName="VMRemote")]
        [String[]]$VMNames,
        [Parameter(ParameterSetName="VMRemote")]
        [PSCredential]$VMCredential,
        [Parameter(ParameterSetName="VMRemote")]
        [CloudEngine.Configurations.EceInterfaceParameters]$Parameters
    )
    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    switch ($Action) {
        "Start"   { $verifyAction = "Running" }
        "Stop"    { $verifyAction = "Stopped" }
        "Restart" { $verifyAction = "Running" }
    }

    $iteration = 0;
    do
    {
        Trace-Execution "$Action service $ServiceName"

        $psToExecute = $Action+"-Service -Name "+$ServiceName
        if ($Force -eq $true) {
            $psToExecute += " -Force"
        }

        $sbActionService = [ScriptBlock]::Create($psToExecute)

        # check the service before attempting to control it
        if ($PSCmdlet.ParameterSetName -eq "Local") {
            $service = ReliableGetService -ServiceName $ServiceName -Local
        } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
            $service = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential
        } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
            $service = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters
        }

        # validate the state
        if ($service.Status -eq $verifyAction) {
            Trace-Execution "Service $($ServiceName) is already $($service.Status)"
            return
        }

        $service.Status
        if ($service.Status -imatch "Pending")
        {
            Trace-Execution "Service $($ServiceName) is in pending state: $($service.Status). Will retry."
        }
        else
        {
            try {
                #Perform the action
                if ($PSCmdlet.ParameterSetName -eq "Local") {
                    $service = Invoke-Command -ScriptBlock $sbActionService
                } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
                    $service = Invoke-Command -ComputerName $Hostnames -Credential $Credential -ScriptBlock $sbActionService
                } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
                    $service = Invoke-PSDirectOnVM -Parameters $Parameters -VMNames $VMNames -VMCredential $VMCredential -ScriptBlock $sbActionService
                }
            } catch {
                Trace-Error "Trying to $Action $ServiceName throw $($_)"
                throw
            }

            # check service status
            if ($PSCmdlet.ParameterSetName -eq "Local") {
                $serviceVerify = ReliableGetService -ServiceName $ServiceName -Local
            } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
                $serviceVerify = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential
            } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
                $serviceVerify = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters
            }

            # validate the state
            if ($serviceVerify.Status -eq $verifyAction) {
                Trace-Execution "Service $($ServiceName) is in desired state: $($verifyAction)"
                break
            }
        }

        $iteration++
        if ($iteration -eq $MaxRetries) {
            Trace-Error "Couldn't $Action the service"
            throw
        }
        Start-Sleep $Delay
        Trace-Execution "Retrying, iteration $iteration"
    } while ($true);
}

function ReliableStopService{
    param (
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Parameter(Mandatory = $true)]
        [String] $ServiceName,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Boolean] $Force = $false,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Boolean] $DisableService = $false,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int32] $MaxRetries = 10,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int] $Delay = 10,
        [Parameter(ParameterSetName="Local")]
        [Switch] $Local,
        [Parameter(ParameterSetName="Remote")]
        [String[]]$Hostnames,
        [Parameter(ParameterSetName="Remote")]
        [PSCredential]$Credential,
        [Parameter(ParameterSetName="VMRemote")]
        [String[]]$VMNames,
        [Parameter(ParameterSetName="VMRemote")]
        [PSCredential]$VMCredential,
        [Parameter(ParameterSetName="VMRemote")]
        [CloudEngine.Configurations.EceInterfaceParameters]$Parameters
    )

    if ($PSCmdlet.ParameterSetName -eq "Local") {
        $service = ReliableGetService -ServiceName $ServiceName -Local
    } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
        $service = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential
    } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
        $service = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters
    }

   # check if we need to disable the service
   if (($DisableService -eq $true) -and ($service.StartType -ne "Disabled")) {
        if ($PSCmdlet.ParameterSetName -eq "Local") {
            ReliableSetService -ServiceName $ServiceName -StartupType "Disabled" -Local -MaxRetries $MaxRetries -Delay $Delay
        } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
            ReliableSetService -ServiceName $ServiceName -StartupType "Disabled" -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay
        } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
            ReliableSetService -ServiceName $ServiceName -StartupType "Disabled" -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay
        }
    }

    # ReliableActionService will handle if the service is already in the requested state.
    if ($PSCmdlet.ParameterSetName -eq "Local") {
        ReliableActionService -ServiceName $ServiceName -Action "Stop" -Force $Force -Local -MaxRetries $MaxRetries -Delay $Delay
    } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
        ReliableActionService -ServiceName $ServiceName -Action "Stop" -Force $Force -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay
    } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
        ReliableActionService -ServiceName $ServiceName -Action "Stop" -Force $Force -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay
    }
}

function ReliableRestartService{
    param (
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Parameter(Mandatory = $true)]
        [String] $ServiceName,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Boolean] $Force = $false,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int32] $MaxRetries = 10,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int] $Delay = 10,
        [Parameter(ParameterSetName="Local")]
        [Switch] $Local,
        [Parameter(ParameterSetName="Remote")]
        [String[]]$Hostnames,
        [Parameter(ParameterSetName="Remote")]
        [PSCredential]$Credential,
        [Parameter(ParameterSetName="VMRemote")]
        [String[]]$VMNames,
        [Parameter(ParameterSetName="VMRemote")]
        [PSCredential]$VMCredential,
        [Parameter(ParameterSetName="VMRemote")]
        [CloudEngine.Configurations.EceInterfaceParameters]$Parameters
    )


    if ($PSCmdlet.ParameterSetName -eq "Local") {
        ReliableActionService -ServiceName $ServiceName -Action "Restart" -Force $Force -Local -MaxRetries $MaxRetries -Delay $Delay
    } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
        ReliableActionService -ServiceName $ServiceName -Action "Restart" -Force $Force -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay
    } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
        ReliableActionService -ServiceName $ServiceName -Action "Restart" -Force $Force -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay
    }
}

function ReliableStartService{
    param (
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Parameter(Mandatory = $true)]
        [String] $ServiceName,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Boolean] $AutomaticService = $false,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int32] $MaxRetries = 10,
        [Parameter(ParameterSetName="Local")]
        [Parameter(ParameterSetName="Remote")]
        [Parameter(ParameterSetName="VMRemote")]
        [Int] $Delay = 10,
        [Parameter(ParameterSetName="Local")]
        [Switch] $Local,
        [Parameter(ParameterSetName="Remote")]
        [String[]]$Hostnames,
        [Parameter(ParameterSetName="Remote")]
        [PSCredential]$Credential,
        [Parameter(ParameterSetName="VMRemote")]
        [String[]]$VMNames,
        [Parameter(ParameterSetName="VMRemote")]
        [PSCredential]$VMCredential,
        [Parameter(ParameterSetName="VMRemote")]
        [CloudEngine.Configurations.EceInterfaceParameters]$Parameters
    )

    if ($PSCmdlet.ParameterSetName -eq "Local") {
        $service = ReliableGetService -ServiceName $ServiceName -Local
    } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
        $service = ReliableGetService -ServiceName $ServiceName -Hostnames $Hostnames -Credential $Credential
    } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
        $service = ReliableGetService -ServiceName $ServiceName -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters
    }

    # check if we need to change to Automatic service
    if (($AutomaticService -eq $true) -and ($service.StartType -ne "Automatic")) {
        if ($PSCmdlet.ParameterSetName -eq "Local") {
            ReliableSetService -ServiceName $ServiceName -StartupType "Automatic" -Local -MaxRetries $MaxRetries -Delay $Delay
        } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
            ReliableSetService -ServiceName $ServiceName -StartupType "Automatic" -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay
        } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
            ReliableSetService -ServiceName $ServiceName -StartupType "Automatic" -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay
        }
    }

    # ReliableActionService will handle if the service is already in the requested state.
    if ($PSCmdlet.ParameterSetName -eq "Local") {
        ReliableActionService -ServiceName $ServiceName -Action "Start" -Local -MaxRetries $MaxRetries -Delay $Delay
    } elseif ($PSCmdlet.ParameterSetName -eq "Remote") {
        ReliableActionService -ServiceName $ServiceName -Action "Start" -Hostnames $Hostnames -Credential $Credential -MaxRetries $MaxRetries -Delay $Delay
    } elseif ($PSCmdlet.ParameterSetName -eq "VMRemote") {
        ReliableActionService -ServiceName $ServiceName -Action "Start" -VMNames $VMNames -VMCredential $VMCredential -Parameters $Parameters -MaxRetries $MaxRetries -Delay $Delay
    }
}

function Invoke-ScriptBlockWithRetries
{
    [CmdletBinding(DefaultParameterSetName="All")]
    param(
        [ScriptBlock]
        [parameter(Mandatory = $true)]
        $ScriptBlock,

        [Int32]
        [parameter(Mandatory = $false)]
        $RetryTimes = 10,

        [Int32]
        [parameter(Mandatory = $false)]
        $RetrySleepTimeInSeconds = 10,

        [Switch]
        [parameter(Mandatory = $false)]
        $InDisconnectedSession=$false,

        [string]
        [parameter(Mandatory = $false, ParameterSetName="ByComputerName")]
        $ComputerName = $null,

        [PSCredential]
        [parameter(Mandatory = $false)]
        $Credential = $null,

        [parameter(Mandatory = $false, ParameterSetName="BySession")]
        $Session = $null,

        [parameter(Mandatory = $false)]
        $ArgumentList = $null
    )
    $ErrorActionPreference = "Stop"

    for($retry = 0; $retry -lt $RetryTimes; ++$retry)
    {
        try
        {
            Write-Verbose "Invoke-ScriptBlockWithRetries: retry #$retry of $RetryTimes, retry sleep time is $RetrySleepTimeInSeconds seconds." -Verbose

            $invokeParams = @{
            }

            $noNewScope = $true
            if ($ComputerName)
            {
                $invokeParams.ComputerName = $ComputerName
                $noNewScope = $false
            }
            elseif ($Session)
            {
                $invokeParams.Session = $Session
                $noNewScope = $false
            }

            if ($Credential)
            {
                $invokeParams.Credential = $Credential
                $noNewScope = $false
            }

            if ($noNewScope)
            {
                $invokeParams.NoNewScope = $true
            }

            if ($InDisconnectedSession)
            {
                $invokeParams.InDisconnectedSession = $true
            }

            if ($ArgumentList)
            {
                $invokeParams.ArgumentList = $ArgumentList
            }

            $invokeParams.ScriptBlock = $ScriptBlock

            return Invoke-Command @invokeParams
        }
        catch
        {
            Write-Verbose "Invoke-ScriptBlockWithRetries failed with exception: $_" -Verbose

            if ($retry -lt ($RetryTimes - 1))
            {
                Start-Sleep -Seconds $RetrySleepTimeInSeconds
            }
            else
            {
                throw
            }
        }
    }
}

function Invoke-ScriptBlockInParallel
{
    [CmdletBinding()]
    param(
        [Parameter(Mandatory=$true)]
        [string[]]
        $ComputerNames,

        [Parameter(Mandatory=$false)]
        [PSCredential]
        $Credential,

        [ScriptBlock]
        [parameter(Mandatory = $true)]
        [ValidateScript({$_ -NotLike '*$Using:*'})]
        $ScriptBlock,

        [Object[]]
        [parameter(Mandatory = $false)]
        $ArgumentList = $null,

        [Int32]
        [parameter(Mandatory = $false)]
        $RetryTimes = 2,

        [Int32]
        [parameter(Mandatory = $false)]
        $RetrySleepTimeInSeconds = 5,

        [Parameter(Mandatory = $false)]
        [string]
        $Authorization = 'Default'
    )

    $ErrorActionPreference = 'Stop'

    $computerNames = $ComputerNames
    $jobs = @()
    $sessions =@()
    $errorMessages = @()

    # Just retry for kicking off the ScriptBlock in Parallel
    for ($retry = 0; $retry -lt $RetryTimes; ++$retry)
    {
        $retryComputers = @()
        $session = $null
        $job = $null

        Trace-Execution "Invoke-ScriptBlockInParallel: retry #$retry of $RetryTimes, retry sleep time is $RetrySleepTimeInSeconds seconds, machines are $computerNames."

        foreach ($computerName in $computerNames)
        {
            try
            {
                if($Credential)
                {
                    $session = New-PSSession -ComputerName $computerName -Credential $Credential -Authentication $Authorization
                }
                else
                {
                    $session = New-PSSession -ComputerName $computerName
                }

                Initialize-ECESession -Session $session
                $job = Invoke-Command -ErrorAction Stop -Session $session -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList -AsJob
                $sessions += $session
                $jobs += $job
            }
            catch
            {
                $retryComputers += $computerName
                $errorMessage = "Machine $computerName failed with error: $($_.Exception.Message)"
                $errorMessages +=  $errorMessage

                if ($session -ne $null)
                {
                    $session | Remove-PSSession -ErrorAction Ignore
                }

                if ($job -ne $null)
                {
                    $job | Remove-Job â€“ErrorAction Ignore
                }
            }
        }

        if (($retryComputers.Count -eq 0) -or ($retry -ge ($RetryTimes -1)))
        {
            Trace-Execution "Invoke-ScriptBlockInParallel exits from retries for kicking off jobs in $retry retries"
            break
        }

        $computerNames = $retryComputers
        Start-Sleep -Second $RetrySleepTimeInSeconds
        $errorMessages = @()
    }

    # Waiting for all jobs to finish
    try
    {
        if ($jobs.Count -eq 0)
        {
            $errorMessage = "Invoke-ScriptBlockInParallel failed: Script could not be invoked after $RetryTimes retries"
            $errorMessages +=  $errorMessage
            throw "Invoke-ScriptBlockInParallel failed: $errorMessages"
        }

        Trace-Execution "Invoke-ScriptBlockInParallel: Waiting for jobs"

        $jobs | Wait-Job | Receive-Job

        Trace-Execution "Invoke-ScriptBlockInParallel: finished jobs"

        $failedJobs =  $jobs | ? { $_.JobStateInfo.State -ne "Completed" }

        foreach ($failedJob in $failedJobs)
        {
            $newErrorMessage = $null
            try
            {
                $newErrorMessage = $failedJob.ChildJobs[0].JobStateInfo.Reason.Message
            }
            catch
            {
            }

            $errorMessages += "Machine $($failedJob.Location): Failed with error: $newErrorMessage"
        }

        if ($errorMessages.Count -gt 0)
        {
            throw "Invoke-ScriptBlockInParallel failed: $errorMessages"
        }
    }
    finally
    {
        $sessions | Remove-PSSession -ErrorAction Ignore
        $jobs | Remove-Job â€“ErrorAction Ignore
    }
}

# Deprecated as iDNS is no longer used
function GetiDNSServersSettings {
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    [string[]] $iDnsServersIPAddress = @()
    $iDnsServersIPAddress
}

<#
.SYNOPSIS
    Create a folder on management file share for host update
 
.DESCRIPTION
    This function creates a folder on management file share for host update
 
.PARAMETER Parameters
    This object is based on the customer manifest. It contains the private information and public
    information of cloud role. This Parameters object is passed down by the deployment engine.
 
#>
function Get-HostUpdateShare
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory=$true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    Trace-ECEScript "Getting the host update share." {
        $physicalMachinesRole = $Parameters.Roles["BareMetal"].PublicConfiguration

        $clusterName = Get-ManagementClusterName $Parameters
        $managementLibraryPath = Get-SharePath $Parameters $physicalMachinesRole.PublicInfo.WindowsUpdateStagingFolder.Path $clusterName
        $managementLibraryPath = Split-Path -Path $managementLibraryPath

        $hostUpdateLogShare = Join-Path $managementLibraryPath 'Logs\HostUpdate'

        if(-not (Test-Path $hostUpdateLogShare))
        {
            Trace-Execution " Create a folder on management file share $hostUpdateLogShare "
            try
            {
                New-Item $hostUpdateLogShare -ItemType Directory -Force | Out-Null
            }
            catch
            {
                throw "Failed to create folder on management file share $hostUpdateLogShare. Error $_"
            }
        }
    }

    return $hostUpdateLogShare
}

# Returns the list of orchestrators that are currently available.
function Get-CurrentOrchestrators
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory=$true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    # Check if the code is running on DVM
    if($Parameters.Roles["DeploymentMachine"])
    {
        $dvmName = $Parameters.Roles["DeploymentMachine"].PublicConfiguration.Nodes.Node | % Name
        if($env:ComputerName -eq $dvmName)
        {
            $dvmIpAddress = $Parameters.Roles["DeploymentMachine"].PublicConfiguration.Nodes.Node | % IPv4Address
            $orchestrator = New-Object -TypeName psobject -Property @{ Name = $dvmName; IPAddress = $dvmIpAddress }
            return @($orchestrator)
        }
    }
    else
    {
        # Check if we only have one host. In that case the code should be running on the same host or the DVM (if performing bare metal POC)
        $physicalMachinesRole = $Parameters.Roles["BareMetal"].PublicConfiguration
        $allPhysicalHosts = @($physicalMachinesRole.Nodes.Node | % Name)
        if( ($allPhysicalHosts.Count -eq 1) -and ($allPhysicalHosts -eq $env:ComputerName) )
        {
            $orchestrator = New-Object -TypeName psobject -Property @{ Name = $env:ComputerName }
            return @( $orchestrator)
        }
    }

    # If the orchestrator is not the DVM or host, then return ERCS VM information
    $orchestrators = @()
    $virtualMachineNodes = $Parameters.Roles["VirtualMachines"].PublicConfiguration.Nodes
    $ercsNodes = $virtualMachineNodes.Node | where { $_.Role -eq "SeedRingServices"}

    foreach($ercsNode in $ercsNodes)
    {
        $name = $ercsNode.Name
        $ercsIpAddress = ($ercsNode.NICs.NIC.IPv4Address -split "/")[0]
        $orchestrator = New-Object -TypeName psobject -Property @{ Name = $name; IPAddress = $ercsIpAddress}
        $orchestrators += $orchestrator
    }
    return @($orchestrators)
}

# Clean up the AD computer object and its DNS record, if it exists
function Remove-ComputerAndDnsRecord
{
    [CmdletBinding()]
    param (
        [string]$Name,
        [string]$DomainFQDN,
        [string]$DNSServerName,
        [PSCredential]$Credential = $null
    )

    if (!$DNSServerName)
    {
        $DNSServerName = (Get-ADDomainController -Discover -DomainName $DomainFQDN).Name
    }

    Trace-Execution "Clean up the AD computer object $Name and its DNS record"
    $computer = Get-ADComputer -Filter {Name -eq $Name}
    if ($computer)
    {
        if (Get-ADObject -SearchBase $computer -SearchScope OneLevel -filter *)
        {
            Remove-ADObject -Identity $computer -Recursive -Confirm:$false
        }
        else
        {
            $computer | Remove-ADComputer -Confirm:$false
        }
    }

    Remove-DnsRecord -DnsZoneName $DomainFQDN -DnsName $Name -DnsServerName $DNSServerName -IPv4Address $true -IPv6Address $true -Credential $Credential
}

#return an list of node which can be connected by PSsession
function Get-AvailableNode
{
    [CmdletBinding()]
    param (
    [Parameter(Mandatory=$true)]
    [string[]]$NodeNames,
    [Parameter(Mandatory=$true)]
    [PSCredential]$Credential,
    [Parameter(Mandatory=$false)]
    [ScriptBlock]$ScriptBlock, #script to validate the current node is available or not with additional check. return $true if it is available.
    [Switch]$Credssp #Whether to use Credssp Authentication for session connected to nodes.
    )
    $ErrorActionPreference = 'Stop'

    $reachableNodes = @()
    foreach($NodeName in $NodeNames)
    {
        try
        {
            if($Credssp.IsPresent)
            {
                $AdditionalParam = @{"Authentication" = "CredSsp"}
            }
            else
            {
                $AdditionalParam = @{}
            }
            $session = New-PSSession -ComputerName $nodeName -Credential $Credential @AdditionalParam
            if($session -eq $null)
            {
                throw "Cannot create session into computer $nodename"
            }
            $result = $true
            if($ScriptBlock -ne $null)
            {
                Invoke-Command -Session $session -ScriptBlock {$ErrorActionPreference = 'Stop'}
                $result = $false
                $result = Invoke-Command -Session $session -ScriptBlock $ScriptBlock
            }

            if($result)
            {
                $reachableNodes += $nodename
            }
        }
        catch
        {
            Trace-Execution "Node $nodename availability validation failed with exception $_"
        }
        finally
        {
            $session | Remove-PSSession -Confirm:$false -ErrorAction Ignore
        }
    }
    return $reachableNodes
}

#Test whether a role can maintain its availability
#It checks a role's available nodes is able to maintain its functionality
#ExcludedNodeNames is the node name for physical machine/the host name for virtual machine. These nodes will be taken as not available
function Test-RoleNodeGroupAvailability
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory=$true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory=$true)]
        [string]$RoleName,

        [Parameter(Mandatory=$true)]
        [PSCredential]$Credential,

        [Parameter(Mandatory=$false)]
        [ScriptBlock]$ScriptBlock, #script to validate the current node is available or not with additional check. return $true if it is available.

        [Switch]$Credssp,#Whether to use Credssp Authentication for session connected to nodes.

        [Parameter(Mandatory=$false)]
        [string[]]$ExcludedNodeNames,

        [Switch]
        $IsHost,

        [Switch]
        $SingleNodeQuorum
    )
    Trace-Execution "Verifying node ring for role $RoleName"

    [Array]$Nodes = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node
    $Nodes += $Parameters.Roles["VirtualMachines"].PublicConfiguration.Nodes.Node
    [Array]$RoleNodes = $Nodes | where { ($_.Role -eq $RoleName) }
    [Array]$RoleProvisionedNodes = $RoleNodes | where { ($_.ProvisioningStatus -eq "Complete")}

    if($IsHost.IsPresent -eq $false)
    {
        $ExcludedNodeNames =  $($RoleNodes | where {(-not [String]::IsNullOrEmpty($_.RefNodeId)) -and ($ExcludedNodeNames -contains $_.RefNodeId) }) |%{$_.Name}
    }

    Trace-Execution "Exclude node $($ExcludedNodeNames -join ',')"

    [Array]$ToVerifiedNodeNames = $($RoleProvisionedNodes.Name | where {$ExcludedNodeNames -notcontains $_})

    Trace-Execution "Verifying node connectivity $($ToVerifiedNodeNames -join ',')"

    if($ToVerifiedNodeNames -and $ToVerifiedNodeNames.Count -ne 0)
    {
        $reachableNodes = Get-AvailableNode -NodeNames $ToVerifiedNodeNames -Credential $Credential -ScriptBlock $ScriptBlock -Credssp:($Credssp.IsPresent)
    }
    else
    {
        $reachableNodes = @()
    }

    Trace-Execution "Nodes $($reachableNodes -join ',') are reachable"

    if($reachableNodes -and $SingleNodeQuorum.IsPresent -and $reachableNodes.Count -gt 0)
    {
        Trace-Execution "After reducing capacity, there still is at least one node running."
        return
    }
    if($reachableNodes -and (-not $SingleNodeQuorum.IsPresent) -and ($reachableNodes.Count * 2 -gt $RoleNodes.Count))
    {
        Trace-Execution "Reducing capacity will maintain a majority of role nodes running."
        return
    }

    throw "Reducing capacity with fewer than or equal to half of initially deployed nodes is not supported for role $RoleName"

}

function Get-CredentialFromStore {
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $true)]
        $SecurityInfoUsers,

        [Parameter(Mandatory = $false)]
        [String]
        $AccountType,

        [Parameter(Mandatory = $false)]
        [String]
        $DomainName
    )
    $VerbosePreference="Continue"
    $ErrorActionPreference = "stop"

    if($AccountType -ne $null)
    {
        $AccountUser = $SecurityInfoUsers | ? Role -EQ $AccountType
        if($AccountUser -ne $null)
        {
            $Credential = $Parameters.GetCredential($AccountUser.Credential)
            if(-not $("$($Credential.UserName)" -like "*\*") -and [String]::IsNullOrEmpty($DomainName) -eq $false)
            {
                $Credential = New-Credential -UserName "$domainName\$($Credential.UserName)" -Password $Credential.GetNetworkCredential().Password
            }
        }
    }
    Write-Output $Credential
}

# Note: This is a temporary implementation to unblock Image Based update
function Update-JEAEndpointsForUpdate {
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    # Update PXE endpoint - Remove this once PXE VM has an updated endpoint.
    # PXE Role
    Trace-Execution "Start - Update module needed by PXEEndpoint on PXE VM."
    $pxeRole = $Parameters.Roles["PXE"].PublicConfiguration
    $pxeServerName = $pxeRole.Nodes.Node.Name

    # PhysicalMachine Role
    $physicalMachinesRole = $Parameters.Roles["BareMetal"].PublicConfiguration
    $allPhysicalHosts = $physicalMachinesRole.Nodes.Node | % Name

    # Skipping nodes which are provided in runtime parameters
    $allPhysicalHosts = Skip-NodesProvidedInRuntimeParameter -Parameters $Parameters -Nodes $allPhysicalHosts

    # VirtualMachine Role
    $virtualMachinesRole = $Parameters.Roles["VirtualMachines"].PublicConfiguration

    $clusterName = Get-ManagementClusterName $Parameters
    $libraryShareNugetStorePath = Get-SharePath $Parameters $virtualMachinesRole.PublicInfo.LibraryShareNugetStoreFolder.Path $clusterName

    $tempNugetDir = Join-Path $env:temp 'AzureStackPXE.Deployment'
    Trace-Execution "Expanding Nuget Microsoft.AzureStack.Solution.Deploy.AzureStackPXE.Deployment at $tempNugetDir"
    Expand-NugetContent -NugetName 'Microsoft.AzureStack.Solution.Deploy.AzureStackPXE.Deployment' `
                        -NugetStorePath $libraryShareNugetStorePath `
                        -SourcePath 'content\Scripts' `
                        -DestinationPath $tempNugetDir

    $sourcePath = "$tempNugetDir\*"
    $destinationPath = "\\$pxeServerName\c$\Program Files\WindowsPowerShell\Modules\Microsoft.AzureStack.Solution.Deploy.AzureStackPXE.Deployment\Scripts"
    Trace-Execution "Copying $sourcePath to $destinationPath"
    New-Item -Path $destinationPath -ItemType Directory -Force -ErrorAction SilentlyContinue
    Copy-Item -Path $sourcePath -Destination $destinationPath -Force
    Remove-Item -Force -Path $tempNugetDir -Recurse

    Trace-Execution "Completed - Update module needed by PXEEndpoint on PXE VM."

    Trace-Execution "Start - Update module needed by NewManagementVMEndpoint on all the hosts"

    $tempNugetDir = Join-Path $env:temp 'NodeUtils_NewManagementVM'
    Expand-NugetContent -NugetName 'Microsoft.AzureStack.Solution.Deploy.Common.NodeUtils' `
                        -NugetStorePath $libraryShareNugetStorePath `
                        -SourcePath 'content\NewManagementVM' `
                        -DestinationPath $tempNugetDir

    foreach ($nodeName in $allPhysicalHosts)
    {
        $sourcePath = "$tempNugetDir\*"
        $destinationPath = "\\$nodeName\c$\Program Files\WindowsPowerShell\Modules\NewManagementVM"
        Trace-Execution "Copying $sourcePath to $destinationPath"
        New-Item -Path $destinationPath -ItemType Directory -Force -ErrorAction SilentlyContinue
        Copy-Item -Path $sourcePath -Destination $destinationPath -Force
    }

    Remove-Item -Force -Path $tempNugetDir -Recurse

    Trace-Execution "Completed - Update module needed by NewManagementVMEndpoint on all the hosts."
}

# Returns credentials needed for various operations during deployment and update.
function Get-DomainCredential
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
    $domainRole = $Parameters.Roles["Domain"].PublicConfiguration
    $securityInfo = $cloudRole.PublicInfo.SecurityInfo
    $domainAdminUser = $securityInfo.DomainUsers.User | ? Role -EQ "DomainAdmin"
    $domainAdminCredential = $Parameters.GetCredential($domainAdminUser.Credential)
    $domainAdminName = $domainAdminCredential.GetNetworkCredential().UserName
    $domainAdminPassword = $domainAdminCredential.GetNetworkCredential().Password
    $domainFqdn = $domainRole.PublicInfo.DomainConfiguration.FQDN
    $domainCredential = New-Credential "$domainFqdn\$domainAdminName" $domainAdminPassword

    return $domainCredential
}

# Returns Temporary Domain Admin credentials.
function Get-TemporaryDomainCredential
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )
    $domainAdminCredential = Get-UserAccountCredential -Parameters $Parameters
    return $domainAdminCredential
}

function Get-UserAccountCredential
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory=$false)]
        [string]
        $UserRole = "DomainAdmin"
    )

    $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration
    $domainRole = $Parameters.Roles["Domain"].PublicConfiguration
    $securityInfo = $cloudRole.PublicInfo.SecurityInfo
    $UserAccount = $securityInfo.DomainUsers.User | ? Role -EQ $UserRole
    $UserAccountCredential = $Parameters.GetCredential($UserAccount.Credential)
    $domainName = $domainRole.PublicInfo.DomainConfiguration.DomainName
    if(-not $("$($UserAccountCredential.UserName)" -like "*\*") -and [String]::IsNullOrEmpty($DomainName) -eq $false)
    {
        $UserAccountCredential = New-Credential -UserName "$domainName\$($UserAccountCredential.UserName)" -Password $UserAccountCredential.GetNetworkCredential().Password
    }
    return $UserAccountCredential
}

# Returns the computer name of the primary DC.
function Get-PrimaryDomainController
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $isOneNode = IsOneNode -Parameters $Parameters
    $restoreContext = Get-RestoreParameters -Parameters $Parameters
    $domainIPMapping = Get-DomainIPMapping -Parameters $Parameters
    $domainAdminCredential = Get-DomainCredential -Parameters $Parameters
    if ($isOneNode -and $restoreContext.RestoreInprogress)
    {
        $primaryDomainController = Get-AvailableADComputerName -AllADComputerName $domainIPMapping.Keys -RemoteServiceCredentials $domainAdminCredential
    }
    else
    {
        $primaryDomainController = Get-AvailableADComputerName -AllADComputerName $domainIPMapping.Keys -RemoteServiceCredentials $domainAdminCredential
    }

    return $primaryDomainController
}

# This function retrieves the DNS Servers.
function Get-DNSServers
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $domainRole = $Parameters.Roles["Domain"].PublicConfiguration
    $virtualMachinesRole = $Parameters.Roles["VirtualMachines"].PublicConfiguration
    $domainControllerName = @($domainRole.Nodes.Node.Name)

    $domainControllers = $virtualMachinesRole.Nodes.Node | ? Name -in $domainControllerName
    $dnsServers = @()
    foreach($nic in $domainControllers.NICs.NIC)
    {
        # IP address is in IP/netmask format, get just the IP address
        $dnsServers += $nic.Ipv4Address.Split('/')[0]
    }

    return $dnsServers
}

# If $Parameters is passed then retry logic will kick in for the exception case
# This is done to handle SF app failover after we fetch the endpoints from the primary
# and before we are done with the SSL binding validation
# the window is very-very small so chances of this are very rare, but do not want to take any chance and handling it
function ValidateCertBinding
{
  [CmdletBinding()]
    param (
    [Parameter(Mandatory=$true)]
    [string]
  $SSLHost,

    [Parameter(Mandatory=$true)]
    [int]
  $Port,

    [Parameter(Mandatory=$true)]
    [string]
  $ExpectedCertThumbprint,

  [Parameter(Mandatory=$false)]
    [string]
  $VMName,

    [Parameter(Mandatory = $false)]
    [CloudEngine.Configurations.EceInterfaceParameters]
    $Parameters,

    [Parameter(Mandatory = $false)]
    $retryIteration
    )
    $ErrorActionPreference = 'Stop'

    # As the path of the dll gets too long if we use it from CloudDeployment\Managed folder, copying it to PS Modules folder until ECE fixes the issue
    $secretManagementBinPathRoot = "$env:systemdrive\Program Files\WindowsPowerShell\Modules\Microsoft.Azurestack.SecretRotation"

    Add-Type -Path "$secretManagementBinPathRoot\Microsoft.AzureStack.SecretManagement.Utilities.dll" -ErrorAction Stop -Verbose:$false | Out-Null
    Add-Type -Path "$secretManagementBinPathRoot\Microsoft.AzureStack.SecretManagement.Tracing.dll" -ErrorAction Stop -Verbose:$false | Out-Null
    Add-Type -Path "$secretManagementBinPathRoot\Microsoft.AzureStack.SecretManagement.CertificateManagement.SSLBoundCertificate.Contract.dll" -ErrorAction Stop -Verbose:$false | Out-Null
    Add-Type -Path "$secretManagementBinPathRoot\Microsoft.AzureStack.SecretManagement.CertificateManagement.SSLBoundCertificate.dll" -ErrorAction Stop -Verbose:$false | Out-Null

  $factory = New-Object Microsoft.AzureStack.SecretManagement.CertificateManagement.SSLBoundCertificate.SSLBoundCertificateFactory
  $sslBoundCertificateWrapper = New-Object Microsoft.AzureStack.SecretManagement.CertificateManagement.SSLBoundCertificate.SSLBoundCertificateWrapper

  if ([string]::IsNullOrEmpty($VMName))
  {
    $SSLBoundCertClass = $factory.CreateSSLBoundCertificate($SSLHost, $Port)
  }
  else
  {
    $SSLBoundCertClass = $factory.CreateSSLBoundCertificate($SSLHost, $VMName, $Port)
  }

  Throw-IfNullOrEmpty -DataToCheck $SSLBoundCertClass -ErrorMessage "Failed to create the instance of SSLBoundCert class"

  $sslBoundCertificate = $sslBoundCertificateWrapper.GetCertBoundToEndpointAsync($SSLBoundCertClass).GetAwaiter().GetResult()
  $certThumbprintBoundToTheEndpoint = $sslBoundCertificate.GetCertHashString()

  Throw-IfNullOrEmpty -DataToCheck $certThumbprintBoundToTheEndpoint -ErrorMessage "Thumbprint for the certificate bound to the endpoint $SSLHost is null"

    Trace-Execution "Found the thumbprint for the certificate bound to the endpoint: $SSLHost it is: $certThumbprintBoundToTheEndpoint"

    #Compare both the thumbprints
    if($ExpectedCertThumbprint -ieq $certThumbprintBoundToTheEndpoint)
    {
        Trace-Execution "Thumbprint bound to the endpoint matched with the one expected from pfx file at rotation location. The thumbprint is: $certThumbprintBoundToTheEndpoint "
    }
    else
    {
        $RetryTimes = 4
        if($Parameters -and $retryIteration -and $retryIteration -lt $RetryTimes)
        {
            # In case SF app failed over after we fetch the endpoints and before validation is done, retrying so that we do not incorrectly report failure
            # Retrying 3 times assumng that if all 3 times failover is the reason then anyway something is broken and we will give up
            $retryIteration++
            Validate-InternalSSLCertBinding -Parameters $Parameters -retryIteration $retryIteration
            Trace-Execution "Retry attempt # $retryIteration "
        }
        else
        {
            throw "The certificate at rotation location is not bound to the endpoint. The one bound to the endpoint has the thumbprint: $certThumbprintBoundToTheEndpoint while the one at rotation location has thumbprint: $ExpectedCertThumbprint"
        }
    }
}

#This function handles the validation of SSL certificate bindings with the endpoints for:
#1. External SSL Certificates with SF apps
#2. External SSL Certificates with highly available endpoints
#3. Internal SSL Certificates with static VIPs and have mapping info in VIPs section of parent Role.xml and certificates info in Role.xml - KeyVault apps
#4. Internal SSL Certificates with static VIPs and have mapping info and certificate info both in VIPs section of Role.xml. These have NO separate renewal path - WAS, WASP
#5. Internal SSL Certificates with static VIPs and have mapping info in Mapping section of VIPs of Parent Role.xml - Some Keyvault apps
function Validate-SSLCertificateBinding
{
    [CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Switch]
        $Internal,

        [Switch]
        $ValidateCurrentCertLocation
    )

    $roleDefinition = $Parameters.Configuration.Role
    $RoleId = $Parameters.Configuration.Role.Id
    $ServiceType = $Parameters.Configuration.Role.ServiceType
    $certRoleDefinition = $Parameters.Roles["CertificateManagement"].PublicConfiguration

    $secureCertPwd = Get-CACertPassword -Parameters $Parameters -ErrorAction Stop -Verbose
    Throw-IfNullOrEmpty -DataToCheck $secureCertPwd -ErrorMessage "Failed to retrieve the password for the certificate."

    # Get external cert information
    $externalCertConfig = $certRoleDefinition.PublicInfo.ExternalCertConfigurations
    $externalCertsInfoList = $externalCertConfig.Certificates.Certificate

    # Get $certsForThisRole for internal SSL certs for given role
    if ($Internal)
    {
        if ($roleDefinition.ServiceType -eq "IIS")
        {
             $VIPForThisRole = $roleDefinition.PublicInfo.VIPs.VIP | Where-object {$_.NetworkId -eq "InternalVip"}
             $certsForThisRole = $VIPForThisRole.PortMapping.Mapping[0].PfxFilePath
        }
        else
        {
            $certsForThisRole = $roleDefinition.PublicInfo.Certificates.Certificate | Where-object {$_.Type -eq "SSL"}
        }
    }
    else
    {
        $certsForThisRole = $externalCertsInfoList.Consumers.Consumer | Where-Object { if ($_.ParentRoleId -eq $null) {$_.RoleId -eq $RoleId} else {$_.ParentRoleId -eq $RoleId} }
    }

    Throw-IfNullOrEmpty -DataToCheck $certsForThisRole -ErrorMessage "Failed to get the list of certificates for the role $RoleId"
    $count = $certsForThisRole.Count
    Trace-Execution "Got certs for this role: $RoleId Count: $count"

    $clusterName = Get-ManagementClusterName $Parameters
    Trace-Execution "Got clusterName $clusterName"

    [string[]] $roleVMs = $roleDefinition.Nodes.Node.Name
    Throw-IfNullOrEmpty -DataToCheck $roleVMs -ErrorMessage "Failed to get the VMs info for the role $RoleId"
    $vmCount = $roleVMs.Count
    Trace-Execution "Found $vmCount VMs for Role: $RoleId"

    $internalDomainFQDN = Get-InternalDomainFQDN -Parameters $Parameters -ErrorAction Stop -Verbose
    Throw-IfNullOrEmpty -DataToCheck $internalDomainFQDN -ErrorMessage "Failed to get Internal Domain FQDN from the role for the role: $RoleId."

    $externalDomainFQDN = Get-ExternalDomainFQDN -Parameters $Parameters -ErrorAction Stop -Verbose
    Throw-IfNullOrEmpty -DataToCheck $externalDomainFQDN -ErrorMessage "Failed to get External Domain FQDN from the role for the role: $RoleId."

    foreach ($consumer in $certsForThisRole)
        {
        if ($Internal)
        {
            if ($roleDefinition.ServiceType -eq "IIS")
            {
                 $renewalFileLocation = $consumer
            }
            else
            {
                if ($ValidateCurrentCertLocation)
                {
                    $renewalFileLocation = $consumer.CertFile
                }
                else
                {
                    $renewalFileLocation = $consumer.CertFileRenewal
                }

            }
        }
        else
        {
             if ($ValidateCurrentCertLocation)
             {
                 $renewalFileLocation = $consumer.Location
             }
             else
             {
                 $renewalFileLocation = $consumer.RotationLocation
             }
        }

        $certLocation = Get-SharePath $Parameters $renewalFileLocation $clusterName

        if(-not $(Test-Path -Path $certLocation))
        {
            throw "Could not find the certificate at this location: $certLocation for the role: $RoleId "
        }
        else
        {
        Trace-Execution "Got certLocation $certLocation"

        $pfxData = Get-PfxData -FilePath $certLocation -Password $secureCertPwd
        Throw-IfNullOrEmpty -DataToCheck $pfxData -ErrorMessage "Failed to get pfxData for the certificate at this location: $certLocation"

        $thumbprintFromRotationPfx = $pfxData.EndEntityCertificates.Thumbprint

        Trace-Execution "Found the thumbprint for the certificate at rotation location: $certLocation. It is: $thumbprintFromRotationPfx "

        Trace-Execution "Validating whether the certificate at rotation location: $certLocation is bound to the endpoint for Role: $RoleId"

        #Get thumbprint for the cert bound to the endpoint
        if (!($roleDefinition.ServiceType -eq "IIS") -or !$internal)
        {
            $vipIdForTheEndpoint = $consumer.VIPId
            Throw-IfNullOrEmpty -DataToCheck $vipIdForTheEndpoint -ErrorMessage "Failed to get VIP ID for the endpoint."
            Trace-Execution "Retrieved the VIP Id for the endpoint: $vipIdForTheEndpoint."
        }

        if ($Internal)
        {
            if ($roleDefinition.ServiceType -eq "IIS")
            {
                 $vipForEndpointFromRole = $VIPForThisRole
            }
            else
            {
                $parentRoleID = $consumer.ParentRoleId
                $parentRoleDefinition = $Parameters.Roles[$parentRoleID].PublicConfiguration
                $vipForEndpointFromRole = $parentRoleDefinition.PublicInfo.VIPs.VIP | Where-Object { $_.Id -eq $vipIdForTheEndpoint }
            }
        }
        else
        {
        $vipForEndpointFromRole = $roleDefinition.PublicInfo.VIPs.VIP | Where-Object { $_.Id -eq $vipIdForTheEndpoint }
        }

        if($consumer.MappingId)
        {
            $mappingId = $consumer.MappingId
            Trace-Execution "It has a MappingId: $mappingId"
            $mapping = $vipForEndpointFromRole.PortMapping.Mapping | Where-Object { $_.Name -eq $mappingId }
            $dnsEndpointFromRole = $mapping.DnsEndpoints.Endpoint
            $mappingsFromRole = $mapping
        }
        else
        {
            $dnsEndpointFromRole = $vipForEndpointFromRole.DnsEndpoints.Endpoint | Where-Object { $_.IsInternal -eq $null }

            # Some roles have different mapping info which have no certs associated. Filtering those out
            $mappingsFromRole = $vipForEndpointFromRole.PortMapping.Mapping | Where-Object { $_.SkipCertValidation -eq $null }
        }

        $endpointFromRole = $dnsEndpointFromRole.Path
        Throw-IfNullOrEmpty -DataToCheck $endpointFromRole -ErrorMessage "Failed to get endpoint for the role: $RoleId."

        Trace-Execution "Retrieved the endpoint: $endpointFromRole."

        Throw-IfNullOrEmpty -DataToCheck $mappingsFromRole -ErrorMessage "Failed to get certificate mapping info for the role: $RoleId and endpoint: $endpointFromRole."

        $endpoint = $endpointFromRole.Replace('*', 'a')

        if ($Internal)
        {
            $DomainFQDN = $internalDomainFQDN
        }
        else
        {
            $DomainFQDN = $externalDomainFQDN
        }

        $sslHost = $endpoint + "." + $DomainFQDN
        Trace-Execution "This is the SSL Host name: $sslHost for the endpoint: $endpoint."

        foreach ($mapping in $mappingsFromRole)
        {
            if ($serviceType -eq "SF")
            {
                # For dynamic VIPs we will go through load balancer and the port applicable is FrontEndPort
                $port = $mapping.FrontEndPort
                Throw-IfNullOrEmpty -DataToCheck $port -ErrorMessage "Failed to get port info for the endpoint: $endpointFromRole"

                Trace-Execution "Validating for SSLHost: $sslHost and port: $port for the endpoint: $endpoint."

                ValidateCertBinding -SSLHost $sslHost -Port $port -ExpectedCertThumbprint $thumbprintFromRotationPfx
            }
            else
            {
                # To validate on each VM, the port applicable is BackEndPort
                $port = $mapping.BackEndPort
                Throw-IfNullOrEmpty -DataToCheck $port -ErrorMessage "Failed to get port info for the endpoint: $endpointFromRole"

                    if ($Internal -and $roleDefinition.ServiceType -eq "IIS")
                    {
                        $renewalCertFileLocation = $mapping.PfxFilePath
                        $certLocation = Get-SharePath $Parameters $renewalCertFileLocation $clusterName

                        if(-not $(Test-Path -Path $certLocation))
                        {
                            throw "Could not find the certificate at this location: $certLocation for the role: $RoleId."
                        }
                        else
                        {
                            Trace-Execution "Got certLocation $certLocation"

                            $pfxDataForInternalIISCert = Get-PfxData -FilePath $certLocation -Password $secureCertPwd
                            Throw-IfNullOrEmpty -DataToCheck $pfxDataForInternalIISCert -ErrorMessage "Failed to get pfxData for the certificate at this location: $certLocation"

                            $thumbprintFromRotationPfx = $pfxDataForInternalIISCert.EndEntityCertificates.Thumbprint

                            Trace-Execution "Found the thumbprint for the certificate at rotation location: $certLocation. It is: $thumbprintFromRotationPfx "

                            Trace-Execution "Validating whether the certificate at rotation location: $certLocation is bound to the endpoint for Role: $RoleId"
                        }
                     }

                     if (($roleDefinition.ServiceType -eq "IIS") -or ($roleDefinition.ServiceType -eq "Standalone"))
                     {
                            # For highly available endpoints, validating binding on every node
                            foreach ($vm in $roleVMs)
                            {
                                Trace-Execution "Validating for SSLHost: $sslHost, VM: $vm and port: $port for the endpoint: $endpoint."

                                ValidateCertBinding -SSLHost $sslHost -Port $port -ExpectedCertThumbprint $thumbprintFromRotationPfx -VMName $vm
                            }
                     }
                     else
                     {
                          # For SF stateless apps we were validating binding on every node, but now changing it due to another change to limit SF app instances to three
                          Trace-Execution "Validating for SSLHost: $sslHost and port: $port for the endpoint: $endpoint."
                          ValidateCertBinding -SSLHost $sslHost -Port $port -ExpectedCertThumbprint $thumbprintFromRotationPfx
                     }
                }
            }
        }
    }
}

# Update Windows DISM features based on the configuration of Role.xml
# Including installing features and removiing unused features
function Update-WindowsDismFeature
{
    [CmdletBinding()]
    Param (
        [Parameter(Mandatory = $false)]
        [object]
        $WindowsFeature,

        [Parameter(Mandatory = $false)]
        [Switch]
        $Online,

        [Parameter(Mandatory = $false)]
        [string]
        $Path,

        [Parameter(Mandatory = $false)]
        [string]
        $WinSxSPath = "$env:windir\WinSxS"
    )

    Trace-Execution "Update Windows DISM features"

    if ($null -ne $WindowsFeature)
    {
        if ($Online)
        {
            $dismFeatures = (Get-WindowsOptionalFeature -Online).FeatureName
        }
        else
        {
            $dismFeatures = (Get-WindowsOptionalFeature -Path $Path).FeatureName
        }

        if ($null -ne $WindowsFeature.Feature.Name)
        {
            $featuresToInstall = $dismFeatures | Where-Object { $_ -in $WindowsFeature.Feature.Name }
            if ($null -ne $featuresToInstall -and $featuresToInstall.Count -gt 0)
            {
                Trace-Execution "Install Windows DISM features: $featuresToInstall"
                if ($Online)
                {
                    Enable-WindowsOptionalFeature -FeatureName $featuresToInstall -Online -All -Source $WinSxSPath -NoRestart
                }
                else
                {
                    Enable-WindowsOptionalFeature -FeatureName $featuresToInstall -Path $path -All
                }
            }
        }

        if ($null -ne $WindowsFeature.RemoveFeature.Name)
        {
            $featuresToRemove = $dismFeatures | Where-Object { $_ -in $WindowsFeature.RemoveFeature.Name }
            if ($null -ne $featuresToRemove -and $featuresToRemove.Count -gt 0)
            {
                Trace-Execution "Remove unused Windows DISM features: $featuresToRemove"
                if ($Online)
                {
                    Disable-WindowsOptionalFeature -FeatureName $featuresToRemove -Online -NoRestart
                }
                else
                {
                    Disable-WindowsOptionalFeature -FeatureName $featuresToRemove -Path $Path
                }
            }
        }
    }
}

function Get-EndpointInfo
{
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $true)]
        [System.String]
        $EndpointUrl
   )
    $endpointUrl = $EndpointUrl.ToUpper()  # e.g. HTTPS://V-XRP03.V.MASD.STBTEST.MICROSOFT.COM:14006
    $pos = $EndpointUrl.ToUpper().IndexOf("HTTPS://")
    $endpoint = $endpointUrl.Substring($pos+8) # gives - V-XRP03.V.MASD.STBTEST.MICROSOFT.COM:14006
    $endpointInfo = $endpoint.Split(":")

    return $endpointInfo
}

 function Validate-EndpointAndPortWithThumbprint
{
[CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $true)]
        $FullEndpointUrl,

        [Parameter(Mandatory = $true)]
        $ThumbprintFromRotationPfx,

        [Parameter(Mandatory = $false)]
        $retryIteration
    )
     $endpoint = Get-EndpointInfo -EndpointUrl $FullEndpointUrl
     $port, $null =  $endpoint[1] -split '/'

     Throw-IfNullOrEmpty -DataToCheck $endpoint -ErrorMessage "Failed to get the endpoint for $FullEndpointUrl"
     Throw-IfNullOrEmpty -DataToCheck $port -ErrorMessage "Failed to get the port for $FullEndpointUrl"

     Trace-Execution "Full endpoint url is: $endpoint and the port is: $port."

     $sslHost = $endpoint[0]

     Throw-IfNullOrEmpty -DataToCheck $sslHost -ErrorMessage "Failed to get the SSL host for $FullEndpointUrl"

   Trace-Execution "This is the SSL Host name: $sslHost for the endpoint: $endpoint."

     ValidateCertBinding -SSLHost $sslHost -Port $port -ExpectedCertThumbprint $ThumbprintFromRotationPfx -Parameters $Parameters -retryIteration $retryIteration
}

function Resolve-EndpointAndPort
{
    [CmdletBinding()]
    param (
            [Parameter(Mandatory = $true)]
            [CloudEngine.Configurations.EceInterfaceParameters]
            $Parameters,

            [Parameter(Mandatory = $true)]
            $ConvertedEndpointInfo,

            [Parameter(Mandatory = $true)]
            $ThumbprintFromRotationPfx,

            [Parameter(Mandatory = $false)]
            $retryIteration
    )
    $restEndpointStr = "OwinListener RestEndpoint" # This is for ACS Migration Service
    $serviceEndpointStr = "ServiceEndpoint" # For most of SF apps
    $serviceManagementEndpointStr = "ServiceManagementEndpoint" # For most of SF apps

    if ($ConvertedEndpointInfo.Count -gt 0)
    {
        foreach ($convertedEndpoint in $ConvertedEndpointInfo)
        {
            $fullRestEndpointUrl = $convertedEndpoint.$restEndpointStr
            if(![string]::IsNullOrEmpty($fullRestEndpointUrl))
            {
                Validate-EndpointAndPortWithThumbprint -Parameters $Parameters -FullEndpointUrl $fullRestEndpointUrl -ThumbprintFromRotationPfx $thumbprintFromRotationPfx -retryIteration $retryIteration
            }

            $fullServiceEndpointUrl = $convertedEndpoint.$serviceEndpointStr
            if(![string]::IsNullOrEmpty($fullServiceEndpointUrl))
            {
                Validate-EndpointAndPortWithThumbprint -Parameters $Parameters -FullEndpointUrl $fullServiceEndpointUrl -ThumbprintFromRotationPfx $thumbprintFromRotationPfx -retryIteration $retryIteration
            }

            $fullServiceManagementEndpointUrl = $convertedEndpoint.$serviceManagementEndpointStr
            if(![string]::IsNullOrEmpty($fullServiceManagementEndpointUrl))
            {
                Validate-EndpointAndPortWithThumbprint -Parameters $Parameters -FullEndpointUrl $fullServiceManagementEndpointUrl -ThumbprintFromRotationPfx $thumbprintFromRotationPfx -retryIteration $retryIteration
            }
        }
    }
    else
    {
        throw  "Failed to get the Endpoint information."
    }

}

function Validate-InternalSSLCertBinding
{
    [CmdletBinding()]
    Param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $false)]
        $retryIteration,

        [Switch]
        $ValidateCurrentCertLocation
    )
    $ErrorActionPreference = 'Stop'
    $roleDefinition = $Parameters.Configuration.Role
    $RoleId = $Parameters.Configuration.Role.Id
    $ServiceType = $Parameters.Configuration.Role.ServiceType
    $securityInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo

    [string] $TimeString = Get-Date -Format "yyyyMMdd-HHmmss"
    $RemoteLOGFILE = "$env:SystemDrive\MASLogs\$($roleDefinition.Id)_$($MyInvocation.MyCommand.Name)_$TimeString.log"

    $secureCertPwd = Get-CACertPassword -Parameters $Parameters -ErrorAction Stop -Verbose
    Throw-IfNullOrEmpty -DataToCheck $secureCertPwd -ErrorMessage "Failed to retrieve the password for the certificate."

    $appName = $roleDefinition.PrivateInfo.ServiceFabricApplication.Name

    # Get the cert information
    $certsForThisRole = $roleDefinition.PublicInfo.Certificates.Certificate | Where-Object { $_.Type -eq "SSL" }
    Throw-IfNullOrEmpty -DataToCheck $certsForThisRole -ErrorMessage "Failed to get the list of certificates for the role $RoleId"

    $count = $certsForThisRole.Count
    if ($count -gt 1)
    {
        throw "Expected only one SSL certificate for role: $RoleId but got $count"
    }

    Trace-Execution "Got SSL certificate for this role: $RoleId"

    if ($ValidateCurrentCertLocation)
    {
        $certFileRenewal = $certsForThisRole.CertFile
    }
    else
    {
        $certFileRenewal = $certsForThisRole.CertFileRenewal
    }

    Trace-Execution "Certificate file for renewal for the role: $RoleId is at: $certFileRenewal"

    $clusterName = Get-ManagementClusterName $Parameters

    Trace-Execution "Got clusterName $clusterName"

    $certLocation = Get-SharePath $Parameters $certFileRenewal $clusterName

    if(-not $(Test-Path -Path $certLocation))
    {
        throw "Could not find the certificate at this location: $certLocation for this role: $RoleId"
    }
    else
    {
        Trace-Execution "Got certLocation $certLocation and validated it"

        $pfxData = Get-PfxData -FilePath $certLocation -Password $secureCertPwd
        Throw-IfNullOrEmpty -DataToCheck $pfxData -ErrorMessage "Failed to get pfxData for the certificate at this location: $certLocation"

        $thumbprintFromRotationPfx = $pfxData.EndEntityCertificates.Thumbprint

        Trace-Execution "Found the thumbprint for the certificate at rotation location: $certLocation. It is: $thumbprintFromRotationPfx "

        Trace-Execution "Validating whether the certificate at rotation location: $certLocation is bound to the endpoint for Role: $RoleId"

        [string[]] $roleVMs = $roleDefinition.Nodes.Node.Name
        Throw-IfNullOrEmpty -DataToCheck $roleVMs -ErrorMessage "Failed to get the VMs info for the role $RoleId"
        $vmCount = $roleVMs.Count
        Trace-Execution "Found $vmCount VMs for Role: $RoleId"

        #Getting endpoint and port info
        $domainFQDN = Get-InternalDomainFQDN -Parameters $Parameters -ErrorAction Stop -Verbose
        Throw-IfNullOrEmpty -DataToCheck $DomainFQDN -ErrorMessage "Failed to get Internal Domain FQDN from the role for the role: $RoleId."

        $credential = Get-CredentialFromStore -Parameters $Parameters -SecurityInfoUsers $securityInfo.DomainUsers.User -AccountType $Parameters.Configuration.Role.PrivateInfo.Accounts.RunAsAccountID -DomainName $domainFqdn

        $firstRoleVM = "$(Get-FirstAvailableNode -Nodes $roleVMs).$domainFqdn"

        Trace-Execution "Getting endpoint and port information for the app: $appName"

        try
        {
            $remoteSession = New-PSSession -ComputerName $firstRoleVM -Credential $credential -Authentication Credssp

            $scriptBlock =
            {
            try
            {
                Start-Transcript -Append -Path $Using:RemoteLOGFILE
                $ErrorActionPreference = "stop"

                Connect-ServiceFabricCluster | Out-Null

                $sfApp = Get-ServiceFabricApplication -ApplicationName $using:appName
                if(!$sfApp)
                {
                    throw "Failed to retrieve Service fabric application for the role: $Using:RoleId."
                }

                Write-Verbose "Got the service fabric application for: $using:appName for the role: $Using:RoleId." -Verbose

                $replicas = @()

                $sfServices = $sfApp | Get-ServiceFabricService

                foreach ($sfService in $sfServices)
                {
                    if ($sfService.ServiceKind -eq 'Stateful')
                    {
                        $replicas += $sfService | Get-ServiceFabricPartition | Get-ServiceFabricReplica |  Where Replicarole -eq Primary
                    }
                    else
                    {
                        $replicas += $sfService | Get-ServiceFabricPartition | Get-ServiceFabricReplica
                    }
                }

                if(!$replicas)
                {
                    throw "Failed to retrieve Service fabric replicas for the application: $sfApp in the role: $Using:RoleId."
                }

                Write-Verbose "Got the service fabric replicas for: $using:appName for the role: $Using:RoleId" -Verbose

                $allReplicasWithReplicaAddress = $replicas | Where-Object {![string]::IsNullOrEmpty($_.ReplicaAddress)}

                $secureReplicasWithReplicaAddress = $allReplicasWithReplicaAddress | Where-Object {$_.ReplicaAddress -match 'https:'}

                if(!$secureReplicasWithReplicaAddress)
                {
                    throw "Failed to retrieve any replicas with replica address for the application: $sfApp in the role: $Using:RoleId."
                }

                Write-Verbose "Got the service fabric replicas with replica address for: $using:appName for the role: $Using:RoleId" -Verbose

                $endpointInfo = $secureReplicasWithReplicaAddress.ReplicaAddress
                if(!$endpointInfo)
                {
                    throw "Failed to retrieve endpoint information from Service fabric replicas for the application: $sfApp in the role: $Using:RoleId."
                }

                Write-Verbose "Got the endpoint information for: $using:appName for the role: $Using:RoleId as $endpointInfo" -Verbose

                try
                {
                     $convertedEndpointInfoCustom = $endpointInfo | ConvertFrom-Json
                     Write-Verbose "Converted the endpoint info from json for endpoint: $endpointInfo" -Verbose
                }
                catch
                {
                    Trace-Error "Failed to convert the endpoint information from json. Please check the json format for the endpoint: $($endpointInfo) throw $($_)"
                    throw
                }

                # Explicitly converting PScustomObject to an array so that when it has one element it returns Count as 1. By default it will not convert it to an array with only one element
                $convertedEndpointArray =  @([PScustomObject]$convertedEndpointInfoCustom)

                $convertedEndpointInfo = $convertedEndpointArray.Endpoints

                if(!$convertedEndpointInfo)
                {
                    throw "Endpoints could not be retrieved for the app. $sfApp in the role: $Using:RoleId."
                }

                Write-Verbose "Got the endpoint information converted from json for: $using:appName for the role: $Using:RoleId" -Verbose

                return $convertedEndpointInfo
            }
            finally
            {
                Stop-Transcript -ErrorAction Ignore
            }
          }

            $dataFromRemote = Invoke-Command -Session $remoteSession -ScriptBlock $scriptBlock

            $dataFromRemoteArray =  @([PScustomObject]$dataFromRemote)

        }
        finally
        {
            $remoteSession | Remove-Pssession -ErrorAction Ignore | Out-Null
        }

        Trace-Execution "Got the endpoint and port information for the app: $appName"

        if(!$retryIteration)
        {
            $retryIteration = 0
        }

        Resolve-EndpointAndPort -Parameters $Parameters -convertedEndpointInfo $dataFromRemoteArray -ThumbprintFromRotationPfx $thumbprintFromRotationPfx -retryIteration $retryIteration
     }
 }

<#
 .Synopsis
  Reliable atomic copy of file.
 
 .Description
  This function performs a reliable copy of a file by using temporary staging copy and hash validation mechanism. This function is designed to be re-entrant.
#>
function Copy-FileAtomic
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true, HelpMessage="Literal path to the source file which is to be copied.")]
        [string]
        $SourceFilePath,

        [Parameter(Mandatory = $true, HelpMessage="Literal path to the destination file where the source file would be copied to.")]
        [string]
        $DestinationFilePath
    )

    $destFile = Split-Path -path $DestinationFilePath -Leaf
    $destFolder = Split-Path -path $DestinationFilePath -Parent

    $newFileName = '{0}.new' -f $destFile
    $prevFileName = '{0}.prev' -f $destFile

    $newPath = Join-Path $destFolder $newFileName
    $prevPath = Join-Path $destFolder $prevFileName

    # Handle code re-entrancy
    if (Test-Path $prevPath)
    {
        if (Test-Path $DestinationFilePath)
        {
            Write-Verbose "Removing stale file: $prevPath"
            Remove-Item -Path $prevPath
        }
        else
        {
            Write-Verbose "Renaming stale file: $prevPath to $DestinationFilePath"
            Rename-Item -Path $prevPath -NewName $DestinationFilePath
        }
    }

    # Copy file to a temp file in destination folder
    Write-Verbose "Copying file $SourceFilePath to $newPath"
    Copy-Item -Path $SourceFilePath -Destination $newPath -Force -ErrorAction Stop

    # Validate copied staging file against original source file
    if ((Get-FileHash $SourceFilePath).hash -ne (Get-FileHash $newPath).hash)
    {
        throw "$newPath is not copied correctly."
    }

    # Handle atomicity and overwrite issues
    $overwrite = Test-Path $DestinationFilePath

    if ($overwrite)
    {
        Write-Verbose "Renaming existing destination file: $DestinationFilePath to $prevPath"
        Rename-Item -Path $DestinationFilePath -NewName $prevPath -ErrorAction Stop
    }

    Write-Verbose "Renaming new file: $newPath to $DestinationFilePath"
    Rename-Item -Path $newPath -NewName $DestinationFilePath -ErrorAction Stop

    if ($overwrite)
    {
        Write-Verbose "Removing stale file: $prevPath"
        Remove-Item -Path $prevPath
    }
}

function Get-ServiceFabricContainerSubnet
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $ErrorActionPreference = "Stop"

    # Attempt to query for the actual subnet value
    $privateInfo = $Parameters.Configuration.Role.PrivateInfo
    # Query for Container Network element
    $containerNetwork = $privateInfo.Configurations.ContainerNetwork
    # Query for Container Network
    if (-not $containerNetwork) { Trace-Error "Couldn't find 'ContainerNetwork' element in 'PrivateInfo/Configurations'" }
    # Query for subnet value
    if ([string]::IsNullOrEmpty($containerNetwork.Subnet)) { Trace-Error "Subnet of 'PrivateInfo/Configurations/ContainerNework' either is not defined, null or empty" }
    # Finally that is subnet value we've been looking for
    return $containerNetwork.Subnet
}

function Get-ServiceFabricTraceSharePath
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $ErrorActionPreference = "Stop"

    $privateInfo = $Parameters.Configuration.Role.PrivateInfo
    $clusterName = Get-ManagementClusterName $Parameters
    $sharePath = Get-SharePath $Parameters $privateInfo.Configurations.TraceRootShare.Path $clusterName
    # Verify share path is actually defined
    if ([string]::IsNullOrEmpty($sharePath)) {
        # NOTE: This error means role definition is missing <TraceRootShare Path="<...>" /> element
        Trace-Error "[$($Parameters.Configuration.Role.Id)]: Cannot get share for saving trace log of Service Fabric cluster"
    }
    return $sharePath
}

function Get-FullyQualifiedDomainName
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $ErrorActionPreference = "Stop"
    $publicInfo = $Parameters.Roles["Domain"].PublicConfiguration.PublicInfo
    return $publicInfo.DomainConfiguration.FQDN
}

function Get-ServiceFabricClusterRunAsAccount
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $ErrorActionPreference = "Stop"

    $fqdn = Get-FullyQualifiedDomainName -Parameters $Parameters
    $privateInfo = $Parameters.Configuration.Role.PrivateInfo
    $accountName = $privateInfo.Configurations.RunAs.AccountName
    # Cluster Credentials
    $clusterAccount = "$fqdn\$accountName$"
    return $clusterAccount
}

<#
   .Synopsis
    Returns the encryption certificate to be used by Service Fabric Central Secret Store (CSS) service.
 
   .Description
    Returns the encryption certificate thumbprint to be used by Service Fabric Central Secret Store (CSS) service by reading
    from the pfx file. Note that this may be different from the one currently in use by CSS.
#>
function Get-SfCssEncryptionCertThumbprintToUse
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    $clusterName = $Parameters.Roles["Cluster"].PublicConfiguration.Clusters.Node | ? IsManagementCluster -eq "true" | Select-Object -ExpandProperty Name

    $roleDefinition = $Parameters.Configuration.Role
    $certFilePathToUse = Get-SharePath $Parameters $roleDefinition.PrivateInfo.Configurations.SfCSS.CertPath $clusterName
    $updateCertFilePath = Get-SharePath $Parameters $roleDefinition.PrivateInfo.Configurations.SfCSS.RenewalCertPath $clusterName

    # If a pfx file exists in the Update directory, use that, otherwise use the one in the Current directory.
    if (Test-Path -Path $updateCertFilePath)
    {
        $certFilePathToUse = $updateCertFilePath
    }

    $certThumbprint = Get-SfCssEncryptionCertThumbprint -Parameters $Parameters -FilePath $certFilePathToUse

    return $certThumbprint
}

<#
   .Synopsis
    Returns the thumbprint for the specified pfx file.
 
   .Description
    Returns the thumbprint for the specified pfx file for use by SF CSS.
#>
function Get-SfCssEncryptionCertThumbprint
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $true)]
        [string]
        $FilePath
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    $roleDefinition = $Parameters.Configuration.Role
    $securityInfo = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.SecurityInfo
    $runasAccount = $securityInfo.DomainUsers.User | ? Role -EQ $roleDefinition.PrivateInfo.Accounts.RunAsAccountID
    $runasAccountCredential = $Parameters.GetCredential($runasAccount.Credential)

    # Explicitly storing node names as an array, because in OneNode environment, PowerShell would treat it as an object instead
    # since there is only one XRP node.
    [string[]] $nodeNames = $roleDefinition.Nodes.Node | ForEach-Object {$_.Name}
    $nodeName = $nodeNames[0]

    # SF CSS certificate generated in 2008 release was protected exclusively to the FileCopyAgent gMSA, so we need to execute
    # Get-PfxData against the certificate using that account. As we can't directly execute an Invoke-Command against
    # a gMSA account, we will use a scheduled task.
    # TODO: Certificates generated in 2104 release will be protected to Domain Admins + FCA gMSA, so we can
    # simplify this logic to directly call Get-PfxData post-2104.
    $domainName = $Parameters.Roles["Domain"].PublicConfiguration.PublicInfo.DomainConfiguration.DomainName
    $serviceAccount = "AzS-Xrp-Fca"
    $certProtectToUser   = '{0}\{1}$' -f $domainName, $serviceAccount

    $getThumbprintScriptBlock =
    {
        $taskName = "GetSfCssEncryptionCertThumbprint-$(New-Guid)"
        $thumbprintFile = "C:\temp\ServiceFabricCssEncryption.txt"
        $SCHED_S_TASK_RUNNING = 267009
        $SCHED_S_TASK_QUEUED = 267045

        Unregister-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue -Confirm:$false | Out-Null
        Remove-Item -Path $thumbprintFile -ErrorAction SilentlyContinue | Out-Null

        $scriptBlockString = @"
            (Get-PfxData -FilePath $using:FilePath).EndEntityCertificates.Thumbprint | New-Item -Path "C:\temp\ServiceFabricCssEncryption.txt" -Force
            exit
"@

        $scriptBlock = [ScriptBlock]::Create($scriptBlockString)

        try
        {
            $TaskParameters = @{ TaskName = $taskName }
            $TaskParameters['Action'] = New-ScheduledTaskAction -Execute "Powershell.exe" -Argument "-NoExit -Command $scriptBlock"

            Trace-Execution "Executing task to fetch SF CSS encryption certificate thumbprint using user: $using:certProtectToUser"
            $TaskParameters['Principal'] = New-ScheduledTaskPrincipal -UserID $using:certProtectToUser -LogonType Password -RunLevel Highest

            Trace-Execution "Registering scheduled task for getting SF CSS encryption certificate thumbprint."
            $scheduledTask = Register-ScheduledTask @TaskParameters -ErrorAction Stop

            Trace-Execution "Starting scheduled task for getting SF CSS encryption certificate thumbprint. Task name: $taskName"
            $scheduledTask | Start-ScheduledTask -ErrorAction Stop

            do
            {
                Start-Sleep -Seconds 1
                $taskResult = $ScheduledTask | Get-ScheduledTaskInfo
            } while (($taskResult.LastTaskResult -eq $SCHED_S_TASK_RUNNING) -or ($taskResult.LastTaskResult -eq $SCHED_S_TASK_QUEUED))

            Trace-Execution "The scheduled task for getting SF CSS encryption certificate thumbprint completed. TaskResult: $($taskResult.LastTaskResult)"

            Get-Content -Path $thumbprintFile
        }
        catch {
           Trace-Error "Failed to fetch SF CSS encrytion certificate thumbprint. Reason: $_"
        }
        finally {
            Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue | Out-Null
            Remove-Item -Path $thumbprintFile  -ErrorAction SilentlyContinue | Out-Null
        }
    }

    # As we need to execute the script block to fetch the cert thumbprint using FCA's gMSA account, we need to
    # execute it from one of the XRP nodes.
    Trace-Execution "Executing script block to fetch SF CSS encryption certificate thumbprint from $nodeName."
    $session = New-PSSession -ComputerName $nodeName -Credential $runasAccountCredential -Authentication Credssp
    try
    {
        [string]$certThumbprint = Invoke-Command -Session $session -ScriptBlock $getThumbprintScriptBlock
    }
    finally
    {
        Remove-PSSession -Session $session -ErrorAction Ignore | Out-Null
    }

    if ([string]::IsNullOrEmpty($certThumbprint))
    {
        throw "SF CSS certificate thumbprint is null or empty."
    }

    Trace-Execution "Determined the encryption certificate thumbprint for SF CSS to be: '$certThumbprint'"
    return $certThumbprint
}

<#
.SYNOPSIS
#### Reference from nuget in 1907
Returns a structured object containing information about each of the exceptions in an update summary
#>
function Read-SummaryXml {
    Param (
        [parameter(Mandatory = $false)][string] $RoutingJsonPath = $RoutingJsonSharePath,
        [parameter(Mandatory = $false, ParameterSetName = 'Path')][string] $SummaryXMLPath,
        [parameter(Mandatory = $false, ParameterSetName = 'Xml')][xml] $SummaryXML,
        [parameter(Mandatory = $false)][switch] $Routing
    )

    if ($SummaryXMLPath)
    {
        $SummaryXML = Get-Content "$SummaryXMLPath" -ErrorAction Stop
    }

    $foundException = $false

    $Exceptions = $SummaryXML.SelectNodes(".//Exception")

    $returnInformation = @()

    foreach ($exception in $Exceptions)
    {
        if ($exception.ParentNode.RolePath)
        {
            $RolePath = $exception.ParentNode.RolePath | select -first 1
            $Interface = $exception.ParentNode.InterfaceType | select -first 1
            $ExecutionContextRolePath = $null
            $ExecutionContextNode = ""
            $ActionType = $null
            $ActionRolePath = ""
            $curAncestor = $exception
            while ($curAncestor -and (!$ExecutionContextRolePath -or !$ActionType))
            {
                if (!$ExecutionContextRolePath)
                {
                    $ExecutionContextRolePath = $curAncestor.ExecutionContext.Roles.Role.RolePath | select -first 1
                    $ExecutionContextNode = $curAncestor.ExecutionContext.Roles.Role.Nodes.Node.Name | select -first 1
                }

                if (!$ActionType)
                {
                    $ActionType = $curAncestor.ActionType
                    $ActionRolePath = $curAncestor.RolePath
                }

                $curAncestor = $curAncestor.ParentNode
            }
            if ($Routing)
            {
                $returnInformation += @(Get-RoutingInformation -RoutingJsonPath $RoutingJsonPath -Exception $exception -ExecutionContextRolePath $ExecutionContextRolePath -Interface $Interface -RolePath $RolePath)
            }
            else
            {
                $returnInformation += @{
                    'Exception' = $exception.Raw
                    'ExecutionContextNode' = $ExecutionContextNode
                    'ExecutionContextRolePath' = $ExecutionContextRolePath
                    'RolePath' = $RolePath
                    'InterfaceType' = $Interface
                    'ActionType' = $ActionType
                    'ActionRolePath' = $ActionRolePath
                    }
            }
        }
    }

    $returnInformation
}

function Get-FailureListFromMainActionPlan
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $runtimeParameters = $null
    $failureList = $null
    if($Parameters.RunInformation -ne $null -and $Parameters.RunInformation.ContainsKey('RuntimeParameter'))
    {
        $runtimeParameters = $Parameters.RunInformation['RuntimeParameter']
    }
    if($runtimeParameters -ne $null -and $runtimeParameters.ContainsKey('FailureSummaryXml'))
    {
        $failureSummaryXml = $runtimeParameters['FailureSummaryXml']
        $failureList = @(Read-SummaryXml -SummaryXML $failureSummaryXml)
    }
    return $failureList
}

<#
.SYNOPSIS
    A generic wrapper function to send telemetry events.
#>
function Send-TelemetryEvent
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory=$true)]
        [string]
        $ComponentName,

        [Parameter(Mandatory=$true)]
        [string]
        $EventName,

        [Parameter(Mandatory=$true)]
        [int]
        $EventVersion,

        [Parameter(Mandatory=$true)]
        [hashtable]
        $EventData
    )

    $ErrorActionPreference = "Stop"
    Trace-Execution "Telemetry event = Component:[$ComponentName], EventName:[$EventName], EventVersion:[$EventVersion], EventData:$($EventData | Out-String)"

    try
    {
        $telemetryPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Fabric.Health.Telemetry.TelemetryReporter"
        $telemetryDll = Join-Path $telemetryPath "lib\net46\Microsoft.AzureStack.Fabric.Health.Telemetry.TelemetryReporter.dll"
        Import-Module -Name $telemetryDll -Verbose:$false -DisableNameChecking
    }
    catch
    {
        Trace-Warning "Error loading telemetry library: $_"
        return
    }

    try
    {
        Trace-Execution "Sending telemetry event."
        Send-AzureStackTelemetryEvent -Component $ComponentName -Name $EventName -Version $EventVersion -TelemetryEventData $EventData -Verbose
        Trace-Execution "Finish sending telemetry event."
    }
    catch
    {
        Trace-Warning "Error sending telemetry event: $_"
    }
}

<#
.SYNOPSIS
    A generic wrapper function to raise or clear secret rotation alert.
#>

function RaiseOrClear-SRAlerts
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )
    Trace-Execution "Inside RaiseOrClear-SRAlerts method."
    $azureStackNugetPath = Get-ASArtifactPath -NugetName "microsoft.azurestack.fault.sdk"
    $alertBinPath = Join-Path $azureStackNugetPath "lib\net46\"
    $dlls = (Get-ChildItem -Path $alertBinPath -Filter "*.dll").FullName
    foreach ($dll in $dlls) {
        Trace-Execution "Loading DLL ... $dll"
        $ret = [System.Reflection.Assembly]::LoadFile($dll)
    }

    $faultWriter = [Microsoft.AzureStack.Fault.Sdk.FaultWriterFactory]::Create()
    $runtimeParameters = $null
    if($Parameters.RunInformation -ne $null -and $Parameters.RunInformation.ContainsKey('RuntimeParameter'))
    {
        $runtimeParameters = $Parameters.RunInformation['RuntimeParameter']
    }

    if($runtimeParameters -ne $null -and $runtimeParameters.ContainsKey('SummaryXml'))
    {
        $resourceProviderNamespace = "Microsoft.Fabric.Admin"
        $summaryXml = [xml]$runtimeParameters['SummaryXml']
        $actionStatus = $summaryXml.Action.Status
        if ($summaryXml.Action.Type -ieq "ExternalCertRotation")
        {
            $faultTypeId = "ExternalSecretRotation.Actionplan.Critical"
        }
        else
        {
            $faultTypeId = "InternalSecretRotation.Actionplan.Critical"
        }
        $resourceType = "infraRoleInstances"
        $resourceName = $env:ComputerName
        if ($actionStatus -eq "Success")
        {
            Trace-Execution "Secret rotation action plan completed successfully, hence close active alert if any."
            $faultWriter.CloseFault($resourceProviderNamespace,$faultTypeId,$resourceType,$resourceName)
        }
        else
        {
            Trace-Execution "Secret rotation action plan status is $actionStatus hence raising an alert."
            $faultWriter.OpenFault($resourceProviderNamespace,$faultTypeId,$resourceType,$resourceName)
        }
    }
}

<#
.SYNOPSIS
    A generic wrapper function checks whether the current update is pnu or not by comparing minor build version.
#>
function Test-IsMonthlyUpdate
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )
    Trace-Execution "Inside Test-IsMonthlyUpdate method."
    $isPnu = $false
    if ((Test-IsUpdate -Parameters $Parameters) -and !(Test-IsHotfixUpdate -Parameters $Parameters))
    {
        $isPnu = $true
    }
    Trace-Execution "Is it a pnu update :: $isPnu."
    return $isPnu
}

<#
.SYNOPSIS
    A generic wrapper function checks whether the current update is a Hotfix pnu or not by checking the Update Name.
#>
function Test-IsHotfixUpdate
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )
    Trace-Execution "Inside Test-IsHotfixUpdate method."
    $isHotfixPNU = $false
    # In case of update, run time information will be populated
    if($Parameters.RunInformation -ne $null)
    {
        $runtimeParameters = $Parameters.RunInformation['RuntimeParameter']
        if( $runtimeParameters -ne $null -and $runtimeParameters.ContainsKey('UpdateVersion') )
        {
            $updateMetadataXml = [xml]$runtimeParameters['metadata']
            Trace-Execution "UpdateName : $($updateMetadataXml.UpdatePackageManifest.UpdateInfo.UpdateName)"
            if($updateMetadataXml.UpdatePackageManifest.UpdateInfo.UpdateName -match 'AzS Hotfix')
            {
                $isHotfixPNU = $true;
            }
        }
    }
    Trace-Execution "Is it a Hotfix Update :: $isHotfixPNU."
    return $isHotfixPNU
}

<#
.SYNOPSIS
    A helper function to check if the current action plan is an Update action plan.
#>
function Test-IsUpdate
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )
    $updateVersion = Get-InProgressUpdateVersion -Parameters $Parameters
    if ($updateVersion -ne $null)
    {
        return $true
    }
    else
    {
        return $false
    }
}

<#
.SYNOPSIS
    Tests whether the execution context node is a member of the management cluster.
#>
function Test-ManagementClusterNode
{
    [CmdletBinding()]
    [OutputType([System.Boolean])]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters
    )

    $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop

    # Default to true if there is no BareMetal execution context
    $result = $true

    # If there is a BareMetal execution context, compare management cluster name and RefClusterId of the node
    if (
        ($nodeName = Get-ExecutionContextNodeName -Parameters $Parameters) -and
        ((Get-ExecutionContextRoleName -Parameters $Parameters) -eq 'BareMetal')
    )
    {
        $managementClusterName = Get-ManagementClusterName $Parameters
        Trace-Execution "Management cluster name is '$managementClusterName'"
        $bareMetalRole = $Parameters.Roles["BareMetal"].PublicConfiguration
        $nodeClusterName = ($bareMetalRole.Nodes.Node | Where-Object {$_.Name -eq $nodeName}).RefClusterId
        Trace-Execution "Node '$nodeName' cluster is '$nodeClusterName'"
        if ($nodeClusterName -eq $managementClusterName)
        {
            Trace-Execution "Node '$nodeName' is a member of management cluster '$managementClusterName'"
        }
        else
        {
            Trace-Execution "Node '$nodeName' is not a member of management cluster '$managementClusterName'"
            $result = $false
        }
    }

    return $result
}

<#
.SYNOPSIS
    A helper function to Skip nodes which are provided in runtime parameters.
#>
function Skip-NodesProvidedInRuntimeParameter
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters]
        $Parameters,

        [Parameter(Mandatory = $true)]
        [string[]]
        $Nodes
    )

    Trace-Execution "[Skip-NodesProvidedInRuntimeParameter] List of Nodes which are getting filtered $($Nodes -join ',')."

    $unreachableNodesStr = $Parameters.RunInformation['RuntimeParameter']['UnreachableNodes']

    if([string]::IsNullOrEmpty($unreachableNodesStr))
    {
        Trace-Execution "[Skip-NodesProvidedInRuntimeParameter] No nodes are present in the list of unreachable nodes."
        return $Nodes
    }

    Trace-Execution "[Skip-NodesProvidedInRuntimeParameter] List of unreachable nodes: $unreachableNodesStr."

    $unreachableNodes = $unreachableNodesStr -split ','

    $newList = $Nodes | % { $_ -notin $unreachableNodes }

    Trace-Execution "[Skip-NodesProvidedInRuntimeParameter] List of Nodes after Filtering $($newList -join ',')."

    return $newList
}

<#
.SYNOPSIS
    A helper function to get first available node.
#>
function Get-FirstAvailableNode
{
    [CmdletBinding()]
    param
    (
        [Parameter(Mandatory = $true)]
        [string[]]
        $Nodes
    )

    Trace-Execution "[Get-FirstAvailableNode] List of Nodes: $($Nodes -join ',')."

    foreach($node in $Nodes)
    {
        $returnValue = Invoke-Command {"reachable"} -ComputerName $node -ErrorAction Ignore

        if($returnValue -eq "reachable")
        {
            Trace-Execution "[Get-FirstAvailableNode] Node selected with WinRM connectivity: '$node'."
            return $node
        }
        else
        {
            Trace-Execution "[Get-FirstAvailableNode] Node is not reachable via WinRM: '$node'."
        }
    }

    Trace-Warning "[Get-FirstAvailableNode] No node is reachable via WinRM, fall back to Test-NetConnection (ping)."

    foreach($node in $Nodes)
    {
        $returnValue = Test-NetConnection -ComputerName $node -ErrorAction Ignore

        if($returnValue.PingSucceeded)
        {
            Trace-Execution "[Get-FirstAvailableNode] Node selected with Test-NetConnection connectivity: '$node'."
            return $node
        }
        else
        {
            Trace-Execution "[Get-FirstAvailableNode] Node is not reachable via Test-NetConnection: '$node'."
        }
    }

    # In few situations where this code is running on ECE agent then there may be permission issues, to work that around
    # returning first node if none of the nodes are reachable.
    # TODO: Fix the permission issue with ECE agent
    Trace-Warning "[Get-FirstAvailableNode] Fall back to return first node as No node is reachable from the list: $($Nodes -join ',')."
    return $Nodes[0]
}

<#
.SYNOPSIS
    A helper function to check if the cloud admin operation status is update critical.
#>
function Test-CloudStatusUpdateCritical
{
    $ErrorActionPreference = "Stop"
    Import-Module -Name ECEClient -Verbose:$false -DisableNameChecking

    $eceClient = Create-ECEClientWithServiceResolver
    $eceParamsXml = [XML]($eceClient.GetCloudParameters().getAwaiter().GetResult().CloudDefinitionAsXmlString)

    $statusNode = $eceParamsXml.SelectNodes("//Parameter[@Name='CloudAdminOperationStatus']")
    return ($statusNode.Value -eq "UpdateCritical")
}

function Get-AzureStackHostAllVirtualSwitchNames
{
    Param(
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters] $Parameters
    )

    [System.String[]] $physicalMachines = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node | Select-Object -ExpandProperty "Name"
    [PSObject[]] $vmSwitch = Get-VMSwitch -ComputerName $physicalMachines[0] -SwitchType External

    if ((-not $vmSwitch) -or $vmSwitch.Count -eq 0)
    {
        Trace-Error "Cannot get Virtual Switch Name from [ $($physicalMachines[0]) ]"
    }

    [System.String[]] $retVal = $vmSwitch.Name
    return $retVal
}

function Get-AzureStackHostDefaultVirtualSwitchName
{
    Param(
        [Parameter(Mandatory = $true)]
        [CloudEngine.Configurations.EceInterfaceParameters] $Parameters
    )

    [System.String[]] $allSwitches = Get-AzureStackHostAllVirtualSwitchNames -Parameters $Parameters

    return $allSwitches[0]
}

Export-ModuleMember -Function Add-AccountAcls
Export-ModuleMember -Function Add-AccountMemberships
Export-ModuleMember -Function Add-AllVIPs
Export-ModuleMember -Function Add-DnsExternalPtrRecords
Export-ModuleMember -Function Add-DnsRecord
Export-ModuleMember -Function Add-DnsServerResourceRecordCNameRemoteOrLocal
Export-ModuleMember -Function Add-DnsServerResourceRecordRemoteOrLocal
Export-ModuleMember -Function Add-GroupMembership
Export-ModuleMember -Function Add-GuestCluster
Export-ModuleMember -Function Add-IDnsConfiguration
Export-ModuleMember -Function Add-IPAddress
Export-ModuleMember -Function Add-LoadBalancerToNetworkAdapter
Export-ModuleMember -Function Add-LocalAdministrators
Export-ModuleMember -Function Add-MachineMembership
Export-ModuleMember -Function Add-NetworkAdapterToNetwork
Export-ModuleMember -Function Add-ServiceAccountMembership
Export-ModuleMember -Function Add-ServiceAccountSpn
Export-ModuleMember -Function Add-UserMembership
Export-ModuleMember -Function Assert-Service
Export-ModuleMember -Function Clear-RemoteDNSServerCache
Export-ModuleMember -Function ConnectPSSession
Export-ModuleMember -Function ConvertFrom-IPAddress
Export-ModuleMember -Function Convert-IPv4IntToString
Export-ModuleMember -Function Convert-IPv4StringToInt
Export-ModuleMember -Function ConvertTo-IPAddress
Export-ModuleMember -Function ConvertTo-MacAddress
Export-ModuleMember -Function ConvertTo-PrefixLength
Export-ModuleMember -Function ConvertTo-SubnetMask
Export-ModuleMember -Function Copy-FileAtomic
Export-ModuleMember -Function Create-ImportModuleString
Export-ModuleMember -Function Create-LocalJob
Export-ModuleMember -Function Create-RemoteJob
Export-ModuleMember -Function Disable-CredSSP
Export-ModuleMember -Function Disable-RemoteClientCredSSP
Export-ModuleMember -Function Dismount-Wim
Export-ModuleMember -Function Enable-AutoLogon
Export-ModuleMember -Function Enable-CredSSP
Export-ModuleMember -Function Expand-DeploymentArtifacts
Export-ModuleMember -Function Expand-NugetContent
Export-ModuleMember -Function Expand-UpdateContent
Export-ModuleMember -Function Find-LockedFiles
Export-ModuleMember -Function Get-FirstAvailableNode
Export-ModuleMember -Function Get-ActionPlanInstanceID
Export-ModuleMember -Function Get-FullyQualifiedDomainName
Export-ModuleMember -Function Get-AvailableADComputerName
Export-ModuleMember -Function Get-AvailableNode
Export-ModuleMember -Function Get-AvailableServer
Export-ModuleMember -Function Get-BareMetalCredential
Export-ModuleMember -Function Get-BroadcastAddress
Export-ModuleMember -Function Get-CACertPassword
Export-ModuleMember -Function Get-ClusterShare
Export-ModuleMember -Function Get-ClusterShareNames
Export-ModuleMember -Function Get-CredentialFromStore
Export-ModuleMember -Function Get-CurrentOrchestrators
Export-ModuleMember -Function Get-DnsServerResourceRecordRemoteOrLocal
Export-ModuleMember -Function Get-DNSServers
Export-ModuleMember -Function Get-DomainCredential
Export-ModuleMember -Function Get-DomainIPMapping
Export-ModuleMember -Function Get-EndpointAndPort
Export-ModuleMember -Function Get-ExecutionContextClusterName
Export-ModuleMember -Function Get-ExecutionContextNodeName
Export-ModuleMember -Function Get-ExecutionContextRoleName
Export-ModuleMember -Function Get-ExternalDnsMachineName
Export-ModuleMember -Function Get-GatewayAddress
Export-ModuleMember -Function Get-HostUpdateShare
Export-ModuleMember -Function GetiDNSServersSettings
Export-ModuleMember -Function Get-InProgressUpdateVersion
Export-ModuleMember -Function Get-InProgressUpdatePackagePath
Export-ModuleMember -Function Get-InProgressUpdateNugetStore
Export-ModuleMember -Function Get-IsVirtualNetworkAlreadyConfigured
Export-ModuleMember -Function Get-JeaSession
Export-ModuleMember -Function Get-LocalCsvPathFromSharePath
Export-ModuleMember -Function Get-MacAddress
Export-ModuleMember -Function Get-MacAddressString
Export-ModuleMember -Function Get-NCAccessControlList
Export-ModuleMember -Function Get-NCCredential
Export-ModuleMember -Function Get-NCGateway
Export-ModuleMember -Function Get-NCGatewayPool
Export-ModuleMember -Function Get-NCIPPool
Export-ModuleMember -Function Get-NCLoadBalancer
Export-ModuleMember -Function Get-NCLoadbalancerManager
Export-ModuleMember -Function Get-NCLoadBalancerMux
Export-ModuleMember -Function Get-NCLogicalNetwork
Export-ModuleMember -Function Get-NCLogicalNetworkSubnet
Export-ModuleMember -Function Get-NCMACPool
Export-ModuleMember -Function Get-NCNetworkInterface
Export-ModuleMember -Function Get-NCNetworkInterfaceInstanceId
Export-ModuleMember -Function Get-NCNetworkInterfaceResourceId
Export-ModuleMember -Function Get-NCPublicIPAddress
Export-ModuleMember -Function Get-NCServer
Export-ModuleMember -Function Get-NCSwitch
Export-ModuleMember -Function Get-NCVirtualGateway
Export-ModuleMember -Function Get-NCVirtualNetwork
Export-ModuleMember -Function Get-NCVirtualServer
Export-ModuleMember -Function Get-NCVirtualSubnet
Export-ModuleMember -Function Get-NetworkAddress
Export-ModuleMember -Function Get-NetworkDefinitionForCluster
Export-ModuleMember -Function Get-NetworkDefinitions
Export-ModuleMember -Function Get-NetworkMap
Export-ModuleMember -Function Get-NetworkNameForCluster
Export-ModuleMember -Function Get-NugetStorePath
Export-ModuleMember -Function Get-NugetVersions
Export-ModuleMember -Function Get-OfflineDjoinBlob
Export-ModuleMember -Function Get-PortProfileId
Export-ModuleMember -Function Get-PrimaryDomainController
Export-ModuleMember -Function Get-RangeEndAddress
Export-ModuleMember -Function Get-ServiceFabricTraceSharePath
Export-ModuleMember -Function Get-ServiceFabricClusterRunAsAccount
Export-ModuleMember -Function Get-ServiceFabricContainerSubnet
Export-ModuleMember -Function Get-SfCssEncryptionCertThumbprintToUse
Export-ModuleMember -Function Get-SfCssEncryptionCertThumbprint
Export-ModuleMember -Function Get-ScopeRange
Export-ModuleMember -Function Get-ServerResourceId
Export-ModuleMember -Function Get-SharePath
Export-ModuleMember -Function Get-StorageEndpointName
Export-ModuleMember -Function Get-TemporaryDomainCredential
Export-ModuleMember -Function Get-UserAccountCredential
Export-ModuleMember -Function Get-TestLogsPath
Export-ModuleMember -Function Get-VMWSManDiagnostics
Export-ModuleMember -Function Initialize-ECESession
Export-ModuleMember -Function Initialize-NugetScript
Export-ModuleMember -Function Invoke-ECECommand
Export-ModuleMember -Function Invoke-PSDirectOnVM
Export-ModuleMember -Function Invoke-ScriptBlockInParallel
Export-ModuleMember -Function Invoke-ScriptBlockWithRetries
Export-ModuleMember -Function Invoke-WebRequestWithRetries
Export-ModuleMember -Function IsIpPoolRangeValid
Export-ModuleMember -Function IsIpWithinPoolRange
Export-ModuleMember -Function IsOneNode
Export-ModuleMember -Function IsOnVirtualMachine
Export-ModuleMember -Function IsVirtualAzureStack
Export-ModuleMember -Function JSONDelete
Export-ModuleMember -Function JSONGet
Export-ModuleMember -Function JSONPost
Export-ModuleMember -Function Mount-Wim
Export-ModuleMember -Function Mount-WindowsImageWithRetry
Export-ModuleMember -Function New-ACL
Export-ModuleMember -Function New-CimSessionVerify
Export-ModuleMember -Function New-Credential
Export-ModuleMember -Function New-ExecutionContextXmlForNode
Export-ModuleMember -Function New-LoadBalancerVIP
Export-ModuleMember -Function New-LocalUserWrapper
Export-ModuleMember -Function New-NCAccessControlList
Export-ModuleMember -Function New-NCAccessControlListRule
Export-ModuleMember -Function New-NCBgpPeer
Export-ModuleMember -Function New-NCBgpRouter
Export-ModuleMember -Function New-NCBgpRoutingPolicy
Export-ModuleMember -Function New-NCBgpRoutingPolicyMap
Export-ModuleMember -Function New-NCCredential
Export-ModuleMember -Function New-NCGateway
Export-ModuleMember -Function New-NCGatewayPool
Export-ModuleMember -Function New-NCGreTunnel
Export-ModuleMember -Function New-NCIPPool
Export-ModuleMember -Function New-NCIPSecTunnel
Export-ModuleMember -Function New-NCL3Tunnel
Export-ModuleMember -Function New-NCLoadBalancer
Export-ModuleMember -Function New-NCLoadBalancerBackendAddressPool
Export-ModuleMember -Function New-NCLoadBalancerFrontEndIPConfiguration
Export-ModuleMember -Function New-NCLoadBalancerLoadBalancingRule
Export-ModuleMember -Function New-NCLoadBalancerMux
Export-ModuleMember -Function New-NCLoadBalancerMuxPeerRouterConfiguration
Export-ModuleMember -Function New-NCLoadBalancerOutboundNatRule
Export-ModuleMember -Function New-NCLoadBalancerProbe
Export-ModuleMember -Function New-NCLoadBalancerProbeObject
Export-ModuleMember -Function New-NCLogicalNetwork
Export-ModuleMember -Function New-NCLogicalNetworkSubnet
Export-ModuleMember -Function New-NCLogicalSubnet
Export-ModuleMember -Function New-NCMACPool
Export-ModuleMember -Function New-NCNetworkInterface
Export-ModuleMember -Function New-NCPublicIPAddress
Export-ModuleMember -Function New-NCServer
Export-ModuleMember -Function New-NCServerConnection
Export-ModuleMember -Function New-NCServerNetworkInterface
Export-ModuleMember -Function New-NCSlbState
Export-ModuleMember -Function New-NCSwitch
Export-ModuleMember -Function New-NCSwitchPort
Export-ModuleMember -Function New-NCVirtualGateway
Export-ModuleMember -Function New-NCVirtualNetwork
Export-ModuleMember -Function New-NCVirtualServer
Export-ModuleMember -Function New-NCVirtualSubnet
Export-ModuleMember -Function New-NCVpnClientAddressSpace
Export-ModuleMember -Function New-RegistryPropertyNameValue
Export-ModuleMember -Function New-ServiceAccount
Export-ModuleMember -Function New-UserAccount
Export-ModuleMember -Function New-UserGroup
Export-ModuleMember -Function NormalizeIPv4Subnet
Export-ModuleMember -Function PublishAndStartDscConfiguration
Export-ModuleMember -Function PublishAndStartDscForJea
Export-ModuleMember -Function RaiseOrClear-SRAlerts
Export-ModuleMember -Function ReliableActionService
Export-ModuleMember -Function ReliableGetService
Export-ModuleMember -Function ReliableRestartService
Export-ModuleMember -Function ReliableSetService
Export-ModuleMember -Function ReliableStartService
Export-ModuleMember -Function ReliableStopService
Export-ModuleMember -Function Remove-ComputerAndDnsRecord
Export-ModuleMember -Function Remove-DnsRecord
Export-ModuleMember -Function Remove-DnsServerResourceRecordRemoteOrLocal
Export-ModuleMember -Function Remove-LoadBalancerFromNetworkAdapter
Export-ModuleMember -Function Remove-NCAccessControlList
Export-ModuleMember -Function Remove-NCCredential
Export-ModuleMember -Function Remove-NCGateway
Export-ModuleMember -Function Remove-NCGatewayPool
Export-ModuleMember -Function Remove-NCIPPool
Export-ModuleMember -Function Remove-NCLoadBalancer
Export-ModuleMember -Function Remove-NCLoadBalancerMux
Export-ModuleMember -Function Remove-NCLogicalNetwork
Export-ModuleMember -Function Remove-NCMACPool
Export-ModuleMember -Function Remove-NCNetworkInterface
Export-ModuleMember -Function Remove-NCPublicIPAddress
Export-ModuleMember -Function Remove-NCServer
Export-ModuleMember -Function Remove-NCSwitch
Export-ModuleMember -Function Remove-NCVirtualGateway
Export-ModuleMember -Function Remove-NCVirtualNetwork
Export-ModuleMember -Function Remove-NCVirtualServer
Export-ModuleMember -Function Remove-PortProfileId
Export-ModuleMember -Function Remove-UserMembership
Export-ModuleMember -Function Reset-RestartCallback
Export-ModuleMember -Function Resolve-EndpointAndPort
Export-ModuleMember -Function Restart-Machine
Export-ModuleMember -Function Set-DNSForwarder
Export-ModuleMember -Function Set-MacAndIPAddressSingleNode
Export-ModuleMember -Function Set-MacAndIPAddress
Export-ModuleMember -Function Set-NCConnection
Export-ModuleMember -Function Set-NCLoadBalancerManager
Export-ModuleMember -Function Set-PortProfileId
Export-ModuleMember -Function Set-PortProfileIdHelper
Export-ModuleMember -Function Set-RestartCallback
Export-ModuleMember -Function Send-TelemetryEvent
Export-ModuleMember -Function Skip-NodesProvidedInRuntimeParameter
Export-ModuleMember -Function Start-CloudCluster
Export-ModuleMember -Function Start-ParallelWork
Export-ModuleMember -Function Start-ParallelWorkAndWait
Export-ModuleMember -Function Start-Test
Export-ModuleMember -Function Stop-CloudCluster
Export-ModuleMember -Function Test-CloudStatusUpdateCritical
Export-ModuleMember -Function Test-ClusterResourceExist
Export-ModuleMember -Function Test-HostPhysicalDiskSize
Export-ModuleMember -Function Test-IPConnection
Export-ModuleMember -Function Test-NetworkMap
Export-ModuleMember -Function Test-IsMonthlyUpdate
Export-ModuleMember -Function Test-IsHotfixUpdate
Export-ModuleMember -Function Test-IsUpdate
Export-ModuleMember -Function Test-ManagementClusterNode
Export-ModuleMember -Function Test-PSSession
Export-ModuleMember -Function Test-PSSessionConnection
Export-ModuleMember -Function Test-RoleNodeGroupAvailability
Export-ModuleMember -Function Test-SFRingHealth
Export-ModuleMember -Function Test-PrecheckClusterNodeHealth
Export-ModuleMember -Function Test-PrecheckNCClusterNodeHealth
Export-ModuleMember -Function Test-WSManConnection
Export-ModuleMember -Function Test-WSmanForCredSSP
Export-ModuleMember -Function Trace-Error
Export-ModuleMember -Function Trace-Execution
Export-ModuleMember -Function Trace-Warning
Export-ModuleMember -Function Update-JEAEndpointsForUpdate
Export-ModuleMember -Function Update-NCCredential
Export-ModuleMember -Function Update-NCServer
Export-ModuleMember -Function Update-NCVirtualServer
Export-ModuleMember -Function Validate-EndpointAndPortWithThumbprint
Export-ModuleMember -Function Validate-InternalSSLCertBinding
Export-ModuleMember -Function Validate-SSLCertificateBinding
Export-ModuleMember -Function Update-WindowsDismFeature
Export-ModuleMember -Function Wait-ParallelWork
Export-ModuleMember -Function Wait-Result
Export-ModuleMember -Function Wait-VirtualNetwork
Export-ModuleMember -Function Read-SummaryXml
Export-ModuleMember -Function Get-FailureListFromMainActionPlan
Export-ModuleMember -Function Get-AzureStackHostAllVirtualSwitchNames
Export-ModuleMember -Function Get-AzureStackHostDefaultVirtualSwitchName

# SIG # Begin signature block
# MIIoKgYJKoZIhvcNAQcCoIIoGzCCKBcCAQExDzANBglghkgBZQMEAgEFADB5Bgor
# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCD2Hmcts5h2jPSk
# EDHACfx0fTeWeI1XDtqn37nI7Zi2LqCCDXYwggX0MIID3KADAgECAhMzAAADTrU8
# esGEb+srAAAAAANOMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p
# bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI5WhcNMjQwMzE0MTg0MzI5WjB0MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
# AQDdCKiNI6IBFWuvJUmf6WdOJqZmIwYs5G7AJD5UbcL6tsC+EBPDbr36pFGo1bsU
# p53nRyFYnncoMg8FK0d8jLlw0lgexDDr7gicf2zOBFWqfv/nSLwzJFNP5W03DF/1
# 1oZ12rSFqGlm+O46cRjTDFBpMRCZZGddZlRBjivby0eI1VgTD1TvAdfBYQe82fhm
# WQkYR/lWmAK+vW/1+bO7jHaxXTNCxLIBW07F8PBjUcwFxxyfbe2mHB4h1L4U0Ofa
# +HX/aREQ7SqYZz59sXM2ySOfvYyIjnqSO80NGBaz5DvzIG88J0+BNhOu2jl6Dfcq
# jYQs1H/PMSQIK6E7lXDXSpXzAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE
# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUnMc7Zn/ukKBsBiWkwdNfsN5pdwAw
# RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW
# MBQGA1UEBRMNMjMwMDEyKzUwMDUxNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci
# tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG
# CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu
# Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0
# MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAD21v9pHoLdBSNlFAjmk
# mx4XxOZAPsVxxXbDyQv1+kGDe9XpgBnT1lXnx7JDpFMKBwAyIwdInmvhK9pGBa31
# TyeL3p7R2s0L8SABPPRJHAEk4NHpBXxHjm4TKjezAbSqqbgsy10Y7KApy+9UrKa2
# kGmsuASsk95PVm5vem7OmTs42vm0BJUU+JPQLg8Y/sdj3TtSfLYYZAaJwTAIgi7d
# hzn5hatLo7Dhz+4T+MrFd+6LUa2U3zr97QwzDthx+RP9/RZnur4inzSQsG5DCVIM
# pA1l2NWEA3KAca0tI2l6hQNYsaKL1kefdfHCrPxEry8onJjyGGv9YKoLv6AOO7Oh
# JEmbQlz/xksYG2N/JSOJ+QqYpGTEuYFYVWain7He6jgb41JbpOGKDdE/b+V2q/gX
# UgFe2gdwTpCDsvh8SMRoq1/BNXcr7iTAU38Vgr83iVtPYmFhZOVM0ULp/kKTVoir
# IpP2KCxT4OekOctt8grYnhJ16QMjmMv5o53hjNFXOxigkQWYzUO+6w50g0FAeFa8
# 5ugCCB6lXEk21FFB1FdIHpjSQf+LP/W2OV/HfhC3uTPgKbRtXo83TZYEudooyZ/A
# Vu08sibZ3MkGOJORLERNwKm2G7oqdOv4Qj8Z0JrGgMzj46NFKAxkLSpE5oHQYP1H
# tPx1lPfD7iNSbJsP6LiUHXH1MIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq
# hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x
# EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv
# bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
# IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG
# EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG
# A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg
# Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
# CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03
# a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr
# rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg
# OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy
# 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9
# sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh
# dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k
# A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB
# w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn
# Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90
# lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w
# ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o
# ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD
# VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa
# BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny
# bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG
# AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t
# L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV
# HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3
# dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG
# AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl
# AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb
# C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l
# hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6
# I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0
# wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560
# STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam
# ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa
# J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah
# XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA
# 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt
# Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr
# /Xmfwb1tbWrJUnMTDXpQzTGCGgowghoGAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw
# EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN
# aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp
# Z25pbmcgUENBIDIwMTECEzMAAANOtTx6wYRv6ysAAAAAA04wDQYJYIZIAWUDBAIB
# BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO
# MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIBY4q+Bo2vE9dWTtA27T9YeV
# 2lbgCVFlbIrGNK2RyG4jMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A
# cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB
# BQAEggEAG0/K7rO4v9qiRR9oLF6ivaXmVBmYvbC4lCyAOuCtpwc0MU9+2l3ih5eq
# ectRmsegNyPTRpDRrgk58p7pdumJTCkRLbuz0MEDtvoStWavWuFBB4WKHnP0X+IE
# 228ZqL09UNrXENu6N1vjx2rsQMXbcAoiEs2fQcPJXqhtegrbkCljoWv4RD5uTaJv
# IlZdZugip0pUmaimhqSSmfpM8PMdJbSdqFdEQs7pZhVaqNXSa+J8zwvcvWp8wf5p
# dFCZZl7osP2riZtx58XmziZmUxFC6SNGaUWLqXLWu2idMucqvgPEFm+suvlEGIl1
# Tzr1Xm1jP/CcYAfvxKeVWMIpgNNpQqGCF5QwgheQBgorBgEEAYI3AwMBMYIXgDCC
# F3wGCSqGSIb3DQEHAqCCF20wghdpAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFSBgsq
# hkiG9w0BCRABBKCCAUEEggE9MIIBOQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl
# AwQCAQUABCCqu5S1HaJrli/IIUmACQiofsjwDcJcbDh3FNdeZ/ITuQIGZMvn7DZV
# GBMyMDIzMDgwNzIxMzM0OC40MzNaMASAAgH0oIHRpIHOMIHLMQswCQYDVQQGEwJV
# UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE
# ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1l
# cmljYSBPcGVyYXRpb25zMScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046MzcwMy0w
# NUUwLUQ5NDcxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2Wg
# ghHqMIIHIDCCBQigAwIBAgITMwAAAdTk6QMvwKxprAABAAAB1DANBgkqhkiG9w0B
# AQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE
# BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD
# VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAeFw0yMzA1MjUxOTEy
# MjdaFw0yNDAyMDExOTEyMjdaMIHLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2Fz
# aGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENv
# cnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25z
# MScwJQYDVQQLEx5uU2hpZWxkIFRTUyBFU046MzcwMy0wNUUwLUQ5NDcxJTAjBgNV
# BAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggIiMA0GCSqGSIb3DQEB
# AQUAA4ICDwAwggIKAoICAQCYU94tmwIkl353SWej1ybWcSAbu8FLwTEtOvw3uXMp
# a1DnDXDwbtkLc+oT8BNti8t+38TwktfgoAM9N/BOHyT4CpXB1Hwn1YYovuYujoQV
# 9kmyU6D6QttTIKN7fZTjoNtIhI5CBkwS+MkwCwdaNyySvjwPvZuxH8RNcOOB8ABD
# hJH+vw/jev+G20HE0Gwad323x4uA4tLkE0e9yaD7x/s1F3lt7Ni47pJMGMLqZQCK
# 7UCUeWauWF9wZINQ459tSPIe/xK6ttLyYHzd3DeRRLxQP/7c7oPJPDFgpbGB2HRJ
# aE0puRRDoiDP7JJxYr+TBExhI2ulZWbgL4CfWawwb1LsJmFWJHbqGr6o0irW7IqD
# kf2qEbMRT1WUM15F5oBc5Lg18lb3sUW7kRPvKwmfaRBkrmil0H/tv3HYyE6A490Z
# FEcPk6dzYAKfCe3vKpRVE4dPoDKVnCLUTLkq1f/pnuD/ZGHJ2cbuIer9umQYu/Fz
# 1DBreC8CRs3zJm48HIS3rbeLUYu/C93jVIJOlrKAv/qmYRymjDmpfzZvfvGBGUbO
# px+4ofwqBTLuhAfO7FZz338NtsjDzq3siR0cP74p9UuNX1Tpz4KZLM8GlzZLje3a
# HfD3mulrPIMipnVqBkkY12a2slsbIlje3uq8BSrj725/wHCt4HyXW4WgTGPizyEx
# TQIDAQABo4IBSTCCAUUwHQYDVR0OBBYEFDzajMdwtAZ6EoB5Hedcsru0DHZJMB8G
# A1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8GA1UdHwRYMFYwVKBSoFCG
# Tmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUy
# MFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBsBggrBgEFBQcBAQRgMF4w
# XAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2Vy
# dHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3J0MAwG
# A1UdEwEB/wQCMAAwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwgwDgYDVR0PAQH/BAQD
# AgeAMA0GCSqGSIb3DQEBCwUAA4ICAQC0xUPP+ytwktdRhYlZ9Bk4/bLzLOzq+wcC
# 7VAaRQHGRS+IPyU/8OLiVoXcoyKKKiRQ7K9c90OdM+qL4PizKnStLDBsWT+ds1ha
# yNkTwnhVcZeA1EGKlNZvdlTsCUxJ5C7yoZQmA+2lpk04PGjcFhH1gGRphz+tcDNK
# /CtKJ+PrEuNj7sgmBop/JFQcYymiP/vr+dudrKQeStcTV9W13cm2FD5F/XWO37Ti
# +G4Tg1BkU25RA+t8RCWy/IHug3rrYzqUcdVRq7UgRl40YIkTNnuco6ny7vEBmWFj
# cr7Skvo/QWueO8NAvP2ZKf3QMfidmH1xvxx9h9wVU6rvEQ/PUJi3popYsrQKuogp
# hdPqHZ5j9OoQ+EjACUfgJlHnn8GVbPW3xGplCkXbyEHheQNd/a3X/2zpSwEROOcy
# 1YaeQquflGilAf0y40AFKqW2Q1yTb19cRXBpRzbZVO+RXUB4A6UL1E1Xjtzr/b9q
# z9U4UNV8wy8Yv/07bp3hAFfxB4mn0c+PO+YFv2YsVvYATVI2lwL9QDSEt8F0RW6L
# ekxPfvbkmVSRwP6pf5AUfkqooKa6pfqTCndpGT71HyiltelaMhRUsNVkaKzAJrUo
# ESSj7sTP1ZGiS9JgI+p3AO5fnMht3mLHMg68GszSH4Wy3vUDJpjUTYLtaTWkQtz6
# UqZPN7WXhjCCB3EwggVZoAMCAQICEzMAAAAVxedrngKbSZkAAAAAABUwDQYJKoZI
# hvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAw
# DgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24x
# MjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAy
# MDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIyNVowfDELMAkGA1UEBhMC
# VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV
# BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp
# bWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC
# AQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXIyjVX9gF/bErg4r25Phdg
# M/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjoYH1qUoNEt6aORmsHFPPF
# dvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1yaa8dq6z2Nr41JmTamDu6
# GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v3byNpOORj7I5LFGc6XBp
# Dco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pGve2krnopN6zL64NF50Zu
# yjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viSkR4dPf0gz3N9QZpGdc3E
# XzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYrbqgSUei/BQOj0XOmTTd0
# lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlMjgK8QmguEOqEUUbi0b1q
# GFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSLW6CmgyFdXzB0kZSU2LlQ
# +QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AFemzFER1y7435UsSFF5PA
# PBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIurQIDAQABo4IB3TCCAdkw
# EgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIEFgQUKqdS/mTEmr6CkTxG
# NSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMFwGA1UdIARV
# MFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0dHA6Ly93d3cubWlj
# cm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0bTATBgNVHSUEDDAK
# BggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMC
# AYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvX
# zpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20v
# cGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYI
# KwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNydDANBgkqhkiG
# 9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv6lwUtj5OR2R4sQaTlz0x
# M7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZnOlNN3Zi6th542DYunKmC
# VgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1bSNU5HhTdSRXud2f8449
# xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4rPf5KYnDvBewVIVCs/wM
# nosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU6ZGyqVvfSaN0DLzskYDS
# PeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDFNLB62FD+CljdQDzHVG2d
# Y3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/HltEAY5aGZFrDZ+kKNxn
# GSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdUCbFpAUR+fKFhbHP+Crvs
# QWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKiexcdFYmNcP7ntdAoGokL
# jzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTmdHRbatGePu1+oDEzfbzL
# 6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZqELQdVTNYs6FwZvKhggNN
# MIICNQIBATCB+aGB0aSBzjCByzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hp
# bmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jw
# b3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2EgT3BlcmF0aW9uczEn
# MCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjM3MDMtMDVFMC1EOTQ3MSUwIwYDVQQD
# ExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMKAQEwBwYFKw4DAhoDFQAt
# M12Wjo2xxA5sduzB/3HdzZmiSKCBgzCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w
# IFBDQSAyMDEwMA0GCSqGSIb3DQEBCwUAAgUA6HusWTAiGA8yMDIzMDgwNzE3NDYw
# MVoYDzIwMjMwODA4MTc0NjAxWjB0MDoGCisGAQQBhFkKBAExLDAqMAoCBQDoe6xZ
# AgEAMAcCAQACAjdHMAcCAQACAhKlMAoCBQDofP3ZAgEAMDYGCisGAQQBhFkKBAIx
# KDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEAAgMBhqAwDQYJKoZI
# hvcNAQELBQADggEBAHITE8uGuO9UrbNbg6/0phE8qAnlqPLB+BAuaQKkGq7N/FLI
# b261b0KeMJTiQtuqSXKoO4oZaIl94fCGwk1dq3BvwVurvjjSFFkK1yEqHbHVqljg
# tuxV+w/MjvX3BcuRBc5Oq+29u4nQLtrXyLobx0tFjr2NxPLFncG4T2m+qBtzPKP5
# fEv+irZVz+Cu43fac2uo88pL4VcYEe1yjqTKwInWdb017DxjYagpHoSpqK2+J/ty
# 0ZCHKYnbLjL5fKX4VAII4jH3/DcwA9TyVP0JGAZCEQZf3gZcbajmREj89jlHNsOp
# 5gv6qVIZkl5o1mY20MIiuR7KajENxrGo93c8pxoxggQNMIIECQIBATCBkzB8MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNy
# b3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAdTk6QMvwKxprAABAAAB1DAN
# BglghkgBZQMEAgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8G
# CSqGSIb3DQEJBDEiBCBgfLIDUeKpCoHO3pm87kRaMYPQZM1vuuRb3xk5uDf6GjCB
# +gYLKoZIhvcNAQkQAi8xgeowgecwgeQwgb0EIMzqh/rYFKXOlzvWS5xCtPi9aU+f
# BUkxIriXp2WTPWI3MIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldh
# c2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBD
# b3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIw
# MTACEzMAAAHU5OkDL8CsaawAAQAAAdQwIgQg7EOEZOoomDBv8SkbDLdBbzGn9fTn
# Z8ys1vZyNyrK+3IwDQYJKoZIhvcNAQELBQAEggIAdQhf5ncmyIquiVPsKYG3HkgO
# 9B7xMCt/46W9UfH9+IPFt9uncltVtDf0HpkIKbm9k698UgBPheeC1tk+GkX5YOSw
# M1uTZ4I8JvMgnZbzQLhnocqvE6rBsr19rZWIqJUS9eq1Jd+3jyYrC72DEAI9SiNY
# cbECRF3pEpQoal9ijt1knoV02LJwDPUr1YBf9mUhzIsVeS9kBtgI/OzMbCU9nAI9
# /2Lklwkz7cysOpCfKikibIAuMKxZmL+PiXAh5AwbhyiO6nhF30qnErTCh1bFnPoY
# NFpxwt+hPjE1txerGBk5B1oUKG6qaLhzoVGoAA4Fq7hEZK+zVMJzx9br6MJ96dNj
# VXZX42dHWxbKEd63ZHAIF8vXKhQqsMtrk8C4wNnLQOCMlbi8RFRVFSpN2Hhn8KeK
# EbVD4G1wAuKZ2IY9KKq8X4zHFyeNTZJ66La6WR8Z6JIIK+iilGL8zYpyLSGJKAZo
# EwzNxOwOrcza3qJ8CrrTJJps4rb2IcZ8v0PP0vLkfhxSjYwu7WFdKFhugJ5eSnoV
# tH+ImuQRjNzwG/l6/pUsBPAiMd9LDwDWMfORIFnU3CLdHEL9iOecgtg4wCKu/zNQ
# LZgXzJP5LuHumdmz3KjQurjiXTFG5fVXqr2C6F4Q7FEwTJrumOiLznSkUQYnnG8K
# nD1qiZdtgiSpc3DO9ug=
# SIG # End signature block