Test-NetStack.psm1

using module .\helpers\prerequisites.psm1
using module .\helpers\internal.psm1
using module .\helpers\icmp.psm1
using module .\helpers\tcp.psm1
using module .\helpers\ndk.psm1

Function Test-NetStack {
    <#
    .SYNOPSIS
        Test-NetStack performs ICMP, TCP, and RDMA traffic testing of networks. Test-NetStack can help you identify misconfigured networks, hotspots, asymmetry across cluster nodes, and more.

    .DESCRIPTION
        Test-NetStack performs ICMP, TCP, and RDMA traffic testing of networks. Test-NetStack can help you identify misconfigured networks, hotspots, asymmetry across cluster nodes, and more.
        Specifically, Test-NetStack:
        - Performs connectivity mapping across a cluster, specific nodes, or IP targets
        - Stage1: ICMP Connectivity, Reliability, and PMTUD
        - Stage2: TCP Stress 1:1
        - Stage3: RDMA Connectivity
        - Stage4: RDMA Stress 1:1
        - Stage5: RDMA Stress N:1
        - Stage6: RDMA Stress N:N
        - Stage7: *EXPERIMENTAL* RDMA Stress N:1 Parallel

    .PARAMETER Nodes
        - Specifies the machines by DNS Name to test.
        - PowerShell remoting without credentials is required; no credentials are stored or entered into Test-NetStack.
        - Minimum 2 nodes, maximum of 16 nodes.

        If part of a failover cluster, and neither the IPTarget or Node parameters are specified, get-clusternode will be run to attempt to gather nodenames.

    .PARAMETER Stage
        List of stages that specifies the tests to be run by Test-NetStack. By default, all stages will be run.

        Tests will always occur in order of lowest stage first. It is highly recommended that you always run the preceeding tests.

        Currently included stages for Test-NetStack:
        - Stage1: ICMP Connectivity, Reliability, and PMTUD
        - Stage2: TCP Stress 1:1
        - Stage3: RDMA Connectivity
        - Stage4: RDMA Stress 1:1
        - Stage5: RDMA Stress N:1
        - Stage6: RDMA Stress N:N

    .PARAMETER EnableFirewallRules
    * This command is best-effort and may fail for a variety of reasons *

    Works with:
    - The Windows Firewall
    - The built-in firewall rules for ICMP, WinRM, and iWARP
    - NTTTCP rules (by application path) needed for Stage 2

    Note: if you upgrade the module version, firewall rules should be revoked, then re-enabled.

    .PARAMETER RevokeFirewallRules
    * This command is best-effort and may fail for a variety of reasons *

    Works with:
    - The Windows Firewall
    - The built-in firewall rules for ICMP and iWARP
    - NTTTCP rules defined by the EnableFirewallRules parameter
    - Will not disable WinRM

    .PARAMETER OnlyPrerequisites
    Use if you want to review the connectivity map detection. This is useful if you're troubleshooting why some networks are or are not included in testing.

    .PARAMETER OnlyConnectivityMap
    Use if you want to review the connectivity map detection. This is useful if you're troubleshooting why some networks are or are not included in testing.

    .PARAMETER LogPath
    Defines the path for the logfile. By default, this will be in the path of the module under the results folder

    .PARAMETER ContinueOnFailure
    By default, Test-NetStack will stop processing later stages if a failure is incurred during an earlier stage. This switch will continue testing later stages.

    The following lists the dependent stages:
    - Stage 1 -> Stage 2
    - Stage 3 -> Stage 4 -> Stage 5 -> Stage 6

    .EXAMPLE
    Run all tests in the local node's failover cluster. Review results from Stage2 and Stage6
        $Results = Test-NetStack

        $Results.Stage2
        $Results.Stage6

    .EXAMPLE
    4-domain joined nodes; all tests run
        $Results = Test-NetStack -Nodes 'AzStackHCI01', 'AzStackHCI02', 'AzStackHCI03', AzStackHCI04'

    .EXAMPLE
    2-node tests; ICMP and TCP tests only. Review results from Stage1 and Stage2
        $Results = Test-NetStack -MachineList 'AzStackHCI01', 'AzStackHCI02' -Stage 1, 2

        $Results.Stage1
        $Results.Stage2

    .NOTES
        Author: Windows Core Networking team @ Microsoft
        Please file issues on GitHub @ GitHub.com/Microsoft/Test-NetStack

    .LINK
        Networking Blog : https://aka.ms/MSFTNetworkBlog
        HCI Host Guidance : https://docs.microsoft.com/en-us/azure-stack/hci/deploy/network-atc
    #>


    [CmdletBinding(DefaultParameterSetName = 'FullNodeMap')]
    param (
        [Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap'       , position = 0)]
        [Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqNodes'   , position = 0)]
        [Parameter(Mandatory = $false, ParameterSetName = 'OnlyConMapNodes'   , position = 0)]
        [Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesNodes', position = 0)]
        [ValidateScript({[System.Uri]::CheckHostName($_) -eq 'DNS'})]
        [ValidateCount(2, 16)]
        [String[]] $Nodes,

        [Parameter(Mandatory = $true, ParameterSetName = 'IPAddress'            , position = 0)]
        [Parameter(Mandatory = $true, ParameterSetName = 'OnlyPrereqIPTarget'   , position = 0)]
        [Parameter(Mandatory = $true, ParameterSetName = 'OnlyConMapIPTarget'   , position = 0)]
        [Parameter(Mandatory = $true, ParameterSetName = 'RevokeFWRulesIPTarget', position = 0)]
        [ValidateCount(2, 16)]
        [IPAddress[]] $IPTarget,

        [Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap'          , position = 1)]
        [Parameter(Mandatory = $false, ParameterSetName = 'IPAddress'            , position = 1)]
        [Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqNodes'      , position = 1)]
        [Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqIPTarget'   , position = 1)]
        [Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesNodes'   , position = 1)]
        [Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesIPTarget', position = 1)]
        [ValidateSet('1', '2', '3', '4', '5', '6', '7', '8')]
        [Int32[]] $Stage = @('1', '2', '3', '4', '5', '6'),

        [Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap', position = 2)]
        [Parameter(Mandatory = $false, ParameterSetName = 'IPAddress'  , position = 2)]
        [Switch] $EnableFirewallRules = $false,

        [Parameter(Mandatory = $true, ParameterSetName = 'RevokeFWRulesNodes'   , position = 2)]
        [Parameter(Mandatory = $true, ParameterSetName = 'RevokeFWRulesIPTarget', position = 2)]
        [Switch] $RevokeFirewallRules = $false,

        [Parameter(Mandatory = $true, ParameterSetName = 'OnlyPrereqNodes'   , position = 2)]
        [Parameter(Mandatory = $true, ParameterSetName = 'OnlyPrereqIPTarget', position = 2)]
        [Switch] $OnlyPrerequisites = $false,

        [Parameter(Mandatory = $true, ParameterSetName = 'OnlyConMapNodes'   , position = 2)]
        [Parameter(Mandatory = $true, ParameterSetName = 'OnlyConMapIPTarget', position = 2)]
        [Switch] $OnlyConnectivityMap = $false,

        [Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap', position = 3)]
        [Parameter(Mandatory = $false, ParameterSetName = 'IPAddress'  , position = 3)]
        [Parameter(Mandatory = $false)]
        [switch] $ContinueOnFailure = $false,

        [Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap'          , position = 4)]
        [Parameter(Mandatory = $false, ParameterSetName = 'IPAddress'            , position = 4)]
        [Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqNodes'      , position = 4)]
        [Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqIPTarget'   , position = 4)]
        [Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesNodes'   , position = 4)]
        [Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesIPTarget', position = 4)]
        [Switch] $Experimental = $false,

        [Parameter(Mandatory = $false)]
        [String] $LogPath = "$(Join-Path -Path $((Get-Module -Name Test-Netstack -ListAvailable | Select-Object -First 1).ModuleBase) -ChildPath "Results\NetStackResults-$(Get-Date -f yyyy-MM-dd-HHmmss).txt")"
    )
    $ExperimentalStages = @('7', '8')
    $ChosenStages = $Stages | Where-Object {$ExperimentalStages -contains $_}
    if ($Experimental -eq $false -and $ChosenStages.Length -gt 0) {
        Write-Error "The experimental stage(s) $ChosenStages have been selected to be run, but the experimental flag has not been set. Please enable it to run experimental stages."
        "The experimental stage(s) $ChosenStages have been selected to be run, but the experimental flag has not been set. Please enable it to run experimental stages." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        break
    }

    $Global:ProgressPreference = 'SilentlyContinue'

    $LogFileParentPath = Split-Path -Path $LogPath -Parent -ErrorAction SilentlyContinue

    if (-not (Test-Path $LogFileParentPath -ErrorAction SilentlyContinue)) {
        $null = New-Item -Path $LogFileParentPath -ItemType Directory -Force -ErrorAction SilentlyContinue
    }

    $LogFile = New-Item -Path $LogPath -ItemType File -Force -ErrorAction SilentlyContinue

    "Starting Test-NetStack - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

    # Each stages adds their results to this and is eventually returned by this function
    $NetStackResults = New-Object -TypeName psobject

    # Since FullNodeMap is the default, we can check if the customer entered Nodes or IPTarget. If neither, check for cluster membership, and use that for the nodes.
    if ( $PsCmdlet.ParameterSetName -eq 'FullNodeMap' -or $PsCmdlet.ParameterSetName -eq 'OnlyPrereqNodes' -or $PsCmdlet.ParameterSetName -eq 'OnlyConMapNodes' -or $PsCmdlet.ParameterSetName -eq 'RevokeFWRulesNodes' ) {
        if (-not($PSBoundParameters.ContainsKey('Nodes'))) {
            try { $Nodes = (Get-ClusterNode -ErrorAction SilentlyContinue -WarningAction SilentlyContinue).Name }
            catch {
                Write-Host 'To run this cmdlet without parameters, join a cluster then try again. Otherwise, specify the Nodes or IPTarget parameters' -ForegroundColor Red
                "To run this cmdlet without parameters, join a cluster then try again. Otherwise, specify the Nodes or IPTarget parameters." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                break
            }
        }

        If ($EnableFirewallRules) { $TargetInfo, $PrereqStatus = Test-NetStackPrerequisites -Nodes $Nodes -Stage $Stage -EnableFirewallRules }
        else { $TargetInfo, $PrereqStatus = Test-NetStackPrerequisites -Nodes $Nodes -Stage $Stage }
    }
    else { # Function returns both the target information and the results of the prerequisite testing
        If ($EnableFirewallRules) { $TargetInfo, $PrereqStatus = Test-NetStackPrerequisites -IPTarget $IPTarget -Stage $Stage -EnableFirewallRules }
        else { $TargetInfo, $PrereqStatus = Test-NetStackPrerequisites -IPTarget $IPTarget -Stage $Stage }
    }

    if ( $RevokeFirewallRules ) {
        if ($IPTarget) { $Targets = $IPTarget }
        else { $Targets = $Nodes }

        Revoke-FirewallRules -Stage $Stage -Targets $Targets

        Return
    }

    $NetStackResults | Add-Member -MemberType NoteProperty -Name Prerequisites -Value $TargetInfo
    Remove-Variable TargetInfo -ErrorAction SilentlyContinue

    "Prerequisite Test Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
    $NetStackResults.Prerequisites | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
    "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

    if ($OnlyPrerequisites) {
        return $NetStackResults
    }
    elseif ($false -in $PrereqStatus) {
        "Prerequsite tests have failed. Review the NetStack results below for more details." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        $NetStackResults.Prerequisites | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        throw "Prerequsite tests have failed. Review the NetStack results for more details."
    }

    #region Connectivity Maps
    if ($Nodes) { $Mapping = Get-ConnectivityMapping -Nodes $Nodes }
    else        { $Mapping = Get-ConnectivityMapping -IPTarget $IPTarget }

    $TestableNetworks     = Get-TestableNetworksFromMapping     -Mapping $Mapping
    $DisqualifiedNetworks = Get-DisqualifiedNetworksFromMapping -Mapping $Mapping

    # If at least one note property doesn't exist, then no disqualified networks were identified
    if (($DisqualifiedNetworks | Get-Member -MemberType NoteProperty).Count) {
        "Disqualified Networks" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        $DisqualifiedNetworks.PSObject.Properties | ForEach-Object {
            $DisqualificationCategory = $_
            "`r`nDisqualification Category: $($DisqualificationCategory.Name)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            $DisqualificationCategory.Value | ForEach-Object {
                $_.Name | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                $_.Group | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            }
        }
        "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        $NetStackResults | Add-Member -MemberType NoteProperty -Name DisqualifiedNetworks -Value $DisqualifiedNetworks
    }
    else { Remove-Variable -Name DisqualifiedNetworks -ErrorAction SilentlyContinue }

    $NetStackResults | Add-Member -MemberType NoteProperty -Name TestableNetworks -Value $TestableNetworks

    if ($TestableNetworks -eq 'None Available' -and (-not($OnlyConnectivityMap))) {
        Write-Error 'No Testable Networks Found. Aborting Test-NetStack.'
        "No Testable Networks Found. Aborting Test-NetStack." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        return $NetStackResults
    }

    "Testable Networks`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
    $TestableNetworks | ForEach-Object {
        $_.Values | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        $_.Group | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
    }
    "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

    if ($OnlyConnectivityMap) { return $NetStackResults }
    #endregion Connectivity Maps

    $runspaceGroups = Get-RunspaceGroups -TestableNetworks $TestableNetworks

    # Defines the stage requirements - internal.psm1
    $Definitions = [Analyzer]::new()

    $ResultsSummary = New-Object -TypeName psobject
    $StageFailures = 0

    $MaxRunspaces = [int]$env:NUMBER_OF_PROCESSORS * 2

    Switch ( $Stage | Sort-Object ) {
        '1' { # ICMP Connectivity, Reliability, and PMTUD
            $thisStage = $_
            Write-Host "Beginning Stage: $thisStage - Connectivity and PMTUD - $([System.DateTime]::Now)"
            "Stage 1`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Beginning Stage: $thisStage - Connectivity and PMTUD - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

            $ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
            $NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
            $NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }

            $RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
            $RunspacePool.Open()

            $AllJobs = @()
            $StageResults = @()

            $TestableNetworks | ForEach-Object {
                $thisTestableNet = $_

                $thisTestableNet.Group | ForEach-Object {
                    $thisSource = $_
                    $thisSourceResult = @()

                    $thisTestableNet.Group | Where-Object NodeName -ne $thisSource.NodeName | ForEach-Object {
                        $thisTarget = $_

                        $PowerShell = [powershell]::Create()
                        $PowerShell.RunspacePool = $RunspacePool

                        [void] $PowerShell.AddScript({
                            param ( $thisComputerName, $thisSource, $thisTarget, $Definitions, $LogFile )

                            if ($thisSource.NodeName -eq $Env:COMPUTERNAME) {
                                $thisSourceResult = Invoke-ICMPPMTUD -Source $thisSource.IPAddress -Destination $thisTarget.IPAddress
                            }
                            else {
                                $thisSourceResult = Invoke-Command -ComputerName $thisComputerName `
                                                                    -ArgumentList $thisSource.IPAddress, $thisTarget.IPAddress `
                                                                    -ScriptBlock  ${Function:\Invoke-ICMPPMTUD}
                            }

                            $Result = New-Object -TypeName psobject
                            $Result | Add-Member -MemberType NoteProperty -Name SourceHostName -Value $thisComputerName
                            $Result | Add-Member -MemberType NoteProperty -Name Source         -Value $thisSource.IPAddress
                            $Result | Add-Member -MemberType NoteProperty -Name Destination    -Value $thisTarget.IPAddress
                            $Result | Add-Member -MemberType NoteProperty -Name Connectivity   -Value $thisSourceResult.Connectivity
                            $Result | Add-Member -MemberType NoteProperty -Name MTU -Value $thisSourceResult.MTU
                            $Result | Add-Member -MemberType NoteProperty -Name MSS -Value $thisSourceResult.MSS

                            if ($thisSource.NodeName -eq $Env:COMPUTERNAME) {
                                $thisSourceResult = Invoke-ICMPPMTUD -Source $thisSource.IPAddress -Destination $thisTarget.IPAddress -StartBytes $thisSourceMSS -Reliability
                            }
                            else {
                                $thisSourceResult = Invoke-Command -ComputerName $thisComputerName `
                                                                    -ArgumentList $thisSource.IPAddress, $thisTarget.IPAddress, $thisSourceResult.MSS , $null, $true `
                                                                    -ScriptBlock ${Function:\Invoke-ICMPPMTUD}
                            }

                            $TotalSent   = $thisSourceResult.Count
                            $TotalFailed = ($thisSourceResult -eq '-1').Count
                            $SuccessPercentage = ([Math]::Round((100 - (($TotalFailed / $TotalSent) * 100)), 2))

                            $Result | Add-Member -MemberType NoteProperty -Name TotalSent   -Value $TotalSent
                            $Result | Add-Member -MemberType NoteProperty -Name TotalFailed -Value $TotalFailed
                            $Result | Add-Member -MemberType NoteProperty -Name Reliability -Value $SuccessPercentage

                            # -1 (no response) will be ignored for LAT and JIT
                            $Latency = Get-Latency -RoundTripTime ($thisSourceResult -ne -1)
                            $Jitter  = Get-Jitter  -RoundTripTime ($thisSourceResult -ne -1)

                            $Result | Add-Member -MemberType NoteProperty -Name Latency -Value $Latency
                            $Result | Add-Member -MemberType NoteProperty -Name Jitter -Value $Jitter

                            if ($TotalSent -and $SuccessPercentage -and $Latency -and $Jitter -and
                                $Definitions.Reliability.ICMPSent  -and $Definitions.Reliability.ICMPReliability  -and
                                $Definitions.Reliability.ICMPLatency -and $Definitions.Reliability.ICMPJitter) {

                                    if ($TotalSent         -ge $Definitions.Reliability.ICMPSent        -and
                                        $SuccessPercentage -ge $Definitions.Reliability.ICMPReliability -and
                                        $Latency           -le $Definitions.Reliability.ICMPLatency     -and
                                        $Jitter            -le $Definitions.Reliability.ICMPJitter ) {

                                        $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass'
                                    }
                                    else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }
                            }
                            else {
                                $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
                                "ERROR: Data failed to be collected for path ($($thisComputerName)) $($thisSource.IPAddress) -> $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                            }

                            return $Result
                        })

                        $param = @{
                            thisComputerName = $thisSource.NodeName
                            thisSource  = $thisSource
                            thisTarget  = $thisTarget
                            Definitions = $Definitions
                            LogFile     = $LogFile
                        }

                        [void] $PowerShell.AddParameters($param)

                        Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] ($($thisSource.NodeName)) $($thisSource.IPAddress) -> $($thisTarget.IPAddress)"
                        ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] ($($thisSource.NodeName)) $($thisSource.IPAddress) -> $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                        $asyncJobObj = @{ JobHandle = $PowerShell; AsyncHandle = $PowerShell.BeginInvoke() }

                        $AllJobs += $asyncJobObj
                        Remove-Variable Result -ErrorAction SilentlyContinue
                    }
                }
            }

            While ($AllJobs -ne $null) {
                $AllJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
                    $thisJob = $_
                    $StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
                    $thisSourceHostName = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).SourceHostName
                    $thisSource = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Source
                    $thisTarget = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Destination

                    $AllJobs = $AllJobs -ne $thisJob

                    Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] ($thisSourceHostName) $($thisSource) -> $($thisTarget)"
                    ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] ($thisSourceHostName) $($thisSource) -> $($thisTarget)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                }
            }

            $RunspacePool.Close()
            $RunspacePool.Dispose()

            if ('Fail' -in $StageResults.PathStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage1 -Value 'Fail'; $StageFailures++ }
            else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage1 -Value 'Pass' }

            $NetStackResults | Add-Member -MemberType NoteProperty -Name Stage1 -Value $StageResults

            Write-Host "Completed Stage: $thisStage - Connectivity and PMTUD - $([System.DateTime]::Now)"
            "Completed Stage: $thisStage - Connectivity and PMTUD - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Stage 1 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            $StageResults | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        }

        '2' { # TCP Stress 1:1
            if ( $ContinueOnFailure -eq $false ) {
                if ('fail' -in $NetStackResults.Stage1.PathStatus) {

                    $Stage -gt 1 | ForEach-Object {
                        $AbortedStage = $_
                        $NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
                    }

                    Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
                    "Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                    return $NetStackResults
                }
            }

            $thisStage = $_

            "Stage 2`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

            $NTttcpLogPath = Join-Path -Path $LogFileParentPath -ChildPath "\NTttcp"
            foreach ($Node in $Nodes) {
                if ($Node -ne $Env:ComputerName) {
                    $NTttcpLogDir = Invoke-Command -ComputerName $Node { New-Item -Path $Using:NTttcpLogPath -ItemType Directory -Force -ErrorAction SilentlyContinue }   
                }
                else { # Machine is local
                
                    $NTttcpLogDir = New-Item -Path $NTttcpLogPath -ItemType Directory -Force -ErrorAction SilentlyContinue
                }
            }
            
            # Using this variable as a placeholder for potential future debug mode options
            # Set to true for now while monitoring lab runs after switching to NTttcp
            $Sequential = $true
            if ( $Sequential -eq $false ) {
                Write-Host "Beginning Stage: $thisStage - TCP - Parallel - $([System.DateTime]::Now)"
                "Beginning Stage: $thisStage - TCP - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                $ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
                $NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
                $NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }

                $RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
                $RunspacePool.Open()

                $StageResults = @()
                foreach ($group in $runspaceGroups) {
                    $GroupedJobs = @()
                    foreach ($pair in $group) {
                        $PowerShell = [powershell]::Create()
                        $PowerShell.RunspacePool = $RunspacePool

                        [void] $PowerShell.AddScript({
                            param ( $thisComputerName, $thisSource, $thisTarget, $Definitions, $LogFile )

                            $Result = New-Object -TypeName psobject
                            $Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
                            $Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
                            $Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress

                            $thisTargetResult = Invoke-TCP -Receiver $thisTarget -Sender $thisSource

                            $Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisTargetResult.ReceiverLinkSpeedGbps
                            $Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisTargetResult.ReceivedGbps
                            $Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisTargetResult.ReceivedPctgOfLinkSpeed
                            $Result | Add-Member -MemberType NoteProperty -Name MinExpectedPctgOfLinkSpeed -Value $Definitions.TCPPerf.TPUT

                            if ($thisTargetResult.ReceivedPctgOfLinkSpeed -and $Definitions.TCPPerf.TPUT) {
                                if ($thisTargetResult.ReceivedPctgOfLinkSpeed -ge $Definitions.TCPPerf.TPUT) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
                                else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }
                            }
                            else {
                                $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
                                "ERROR: Data failed to be collected for path $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                            }

                            $Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisTargetResult.RawData

                            Return $Result
                        })

                        $param = @{
                            thisComputerName = $pair.Source.NodeName
                            thisSource  = $pair.Source
                            thisTarget  = $pair.Target
                            Definitions = $Definitions
                            LogFile     = $LogFile
                        }

                        [void] $PowerShell.AddParameters($param)

                        Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($pair.Source.IPAddress) -> ($($pair.Target.NodeName)) $($pair.Target.IPAddress)"
                        ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($pair.Source.IPAddress) -> ($($pair.Target.NodeName)) $($pair.Target.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                        $asyncJobObj = @{ JobHandle   = $PowerShell
                                            AsyncHandle = $PowerShell.BeginInvoke() }

                        $GroupedJobs += $asyncJobObj
                    }

                    While ($GroupedJobs -ne $null) {
                        $GroupedJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
                            $thisJob = $_
                            $StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
                            $thisReceiverHostName = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).ReceiverHostName
                            $thisSource = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Sender
                            $thisTarget = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Receiver

                            $GroupedJobs = $GroupedJobs -ne $thisJob

                            Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)"
                            ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                        }
                    }
                }

                $RunspacePool.Close()
                $RunspacePool.Dispose()
            } 
            else { # Run sequentially
                Write-Host "Beginning Stage 2 - TCP - Sequential - $([System.DateTime]::Now)"
                "Beginning Stage 2 - TCP - Sequential - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                $StageResults = @()
                $TestableNetworks | ForEach-Object {
                    $thisTestableNet = $_

                    $thisTestableNet.Group | ForEach-Object {
                        $thisSource = $_
                        $thisSourceResult = @()

                        $thisTestableNet.Group | Where-Object NodeName -ne $thisSource.NodeName | ForEach-Object {
                            $thisTarget = $_

                            $Result = New-Object -TypeName psobject
                            $Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
                            $Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
                            $Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress

                            Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IpAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
                            ":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IpAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                        
                            $thisSourceResult = Invoke-TCP -Receiver $thisTarget -Sender $thisSource -LogDir $NTttcpLogDir

                            Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IpAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
                            ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IpAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                            $Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisSourceResult.ReceiverLinkSpeedGbps
                            $Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.ReceivedGbps
                            $Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisSourceResult.ReceivedPctgOfLinkSpeed
                            $Result | Add-Member -MemberType NoteProperty -Name MinExpectedPctgOfLinkSpeed -Value $Definitions.TCPPerf.TPUT
                        
                            $ThroughputPercentageDec = $Definitions.TCPPerf.TPUT / 100.0
                            $AcceptableThroughput = $thisSourceResult.RawData.MinLinkSpeedbps * $ThroughputPercentageDec

                            if ($thisSourceResult.ReceivedPctgOfLinkSpeed -ge $Definitions.TCPPerf.TPUT) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
                            else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }

                            $Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData

                            $StageResults += $Result
                            Remove-Variable Result -ErrorAction SilentlyContinue
                        }
                    }
                }
            }

            if ('Fail' -in $StageResults.PathStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage2 -Value 'Fail'; $StageFailures++ }
            else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage2 -Value 'Pass' }

            $NetStackResults | Add-Member -MemberType NoteProperty -Name Stage2 -Value $StageResults
            Write-Host "Completed Stage: $thisStage - TCP - $([System.DateTime]::Now)"
            "Completed Stage: $thisStage - TCP - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Stage 2 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            $StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        }

        '3' { # RDMA Connectivity
            $thisStage = $_
            "Stage 3`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

            # Using this variable as a placeholder for potential future debug mode options
            # Set to true for now while NDKPerf does not support parallel connections
            $Sequential = $true
            if ($Sequential -eq $false) { # Run in parallel (re-enable when supported by NDKPerf)
                Write-Host "Beginning Stage: $thisStage - RDMA Ping - Parallel - $([System.DateTime]::Now)"
                "Beginning Stage: $thisStage - RDMA Ping - Parallel - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                $ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
                $NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
                $NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }

                $RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
                $RunspacePool.Open()

                $AllJobs = @()
                $StageResults = @()
                $TestableNetworks | ForEach-Object {
                    $thisTestableNet = $_

                    $thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
                        $thisSource = $_
                        $thisSourceResult = @()

                        $thisTestableNet.Group | Where-Object NodeName -ne $thisSource.NodeName | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
                            $thisTarget = $_

                            $PowerShell = [powershell]::Create()
                            $PowerShell.RunspacePool = $RunspacePool

                            [void] $PowerShell.AddScript({
                                param ( $thisSource, $thisTarget, $Definitions )

                                $Result = New-Object -TypeName psobject
                                $Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
                                $Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
                                $Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress

                                $thisSourceResult = Invoke-NDKPing -Server $thisTarget -Client $thisSource

                                if ($thisSourceResult.ServerSuccess) {
                                    $Result | Add-Member -MemberType NoteProperty -Name Connectivity -Value $true
                                    $Result | Add-Member -MemberType NoteProperty -Name PathStatus   -Value 'Pass'
                                }
                                else {
                                    $Result | Add-Member -MemberType NoteProperty -Name Connectivity -Value $false
                                    $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
                                }

                                Return $Result
                            })

                            $param = @{
                                thisSource  = $thisSource
                                thisTarget  = $thisTarget
                                Definitions = $Definitions
                            }

                            [void] $PowerShell.AddParameters($param)

                            Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($thisSource.IPAddress) -> $($thisTarget.IPAddress)"
                            ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($thisSource.IPAddress) -> $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                            $asyncJobObj = @{ JobHandle   = $PowerShell
                                              AsyncHandle = $PowerShell.BeginInvoke() }

                            $AllJobs += $asyncJobObj
                        }
                    }
                }

                While ($AllJobs -ne $null) {
                    $AllJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
                        $thisJob = $_
                        $StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
                        $thisReceiverHostName = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).ReceiverHostName
                        $thisSource = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Sender
                        $thisTarget = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Receiver

                        $AllJobs = $AllJobs -ne $thisJob

                        Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)"
                        ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                    }
                }

                $RunspacePool.Close()
                $RunspacePool.Dispose()
            } else { # Run sequentially
                Write-Host "Beginning Stage 3 - NDK Ping - Sequential - $([System.DateTime]::Now)"
                "Beginning Stage: $thisStage - RDMA Ping - Sequential - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                $NDKLogPath = Join-Path -Path $LogFileParentPath -ChildPath "\NDK\NDKOutput-$(Get-Date -f yyyy-MM-dd-HHmmss).txt"
                $NDKLog = New-Item -Path $NDKLogPath -ItemType File -Force -ErrorAction SilentlyContinue

                $StageResults = @()
                $TestableNetworks | ForEach-Object {
                    $thisTestableNet = $_

                    $thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
                        $thisSource = $_
                        $thisSourceResult = @()
                    
                        $thisTestableNet.Group | Where-Object NodeName -ne $thisSource.NodeName | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
                            $thisTarget = $_

                            $Result = New-Object -TypeName psobject
                            $Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
                            $Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
                            $Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress

                            Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IPAddress) -> $($thisTarget.IPAddress)"
                            ":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IPAddress) -> $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                            $thisSourceResult = Invoke-NDKPing -Server $thisTarget -Client $thisSource -NDKLog $NDKLog

                            Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
                            ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                            if ($thisSourceResult.ServerSuccess) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
                            else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }

                            $StageResults += $Result
                            Remove-Variable Result -ErrorAction SilentlyContinue
                        }
                    }
                }
            }

            

            if ('Fail' -in $StageResults.PathStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage3 -Value 'Fail'; $StageFailures++ }
            else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage3 -Value 'Pass' }

            $NetStackResults | Add-Member -MemberType NoteProperty -Name Stage3 -Value $StageResults

            Write-Host "Completed Stage: $thisStage - RDMA Ping - $([System.DateTime]::Now)"
            "Completed Stage: $thisStage - RDMA Ping - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Stage 3 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            $StageResults | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        }

        '4' { # RDMA Stress 1:1
            if ( $ContinueOnFailure -eq $false ) {
                if ('fail' -in $NetStackResults.Stage3.PathStatus) {

                    $Stage -ge 4 | ForEach-Object {
                        $AbortedStage = $_
                        $NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
                    }

                    Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
                    "Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                    return $NetStackResults
                }
            }

            $thisStage = $_
            "Stage 4`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            Write-Host "Beginning Stage: $thisStage - RDMA Perf 1:1 - $([System.DateTime]::Now)"
            "Beginning Stage: $thisStage - RDMA Perf 1:1 - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

            # Using this variable as a placeholder for potential future debug mode options
            # Set to true for now while NDKPerf does not support parallel connections
            $Sequential -eq $true
            if ($Sequential -eq $false) { # Run in parallel (re-enable when supported by NDKPerf)
                Write-Host "Beginning Stage: $thisStage - RDMA Perf 1:1 - Parallel - $([System.DateTime]::Now)"
                "Beginning Stage: $thisStage - RDMA Perf 1:1 - Parallel - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                $ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
                $NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
                $NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }

                $RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
                $RunspacePool.Open()

                $StageResults = @()
                foreach ($group in $runspaceGroups) {
                    $GroupedJobs = @()
                    foreach ($pair in ($group | Where {($group.Source.RDMAEnabled) -and ($group.Target.RDMAEnabled)})) {

                        $PowerShell = [powershell]::Create()
                        $PowerShell.RunspacePool = $RunspacePool

                        [void] $PowerShell.AddScript({
                            param ( $thisSource, $thisTarget, $Definitions )

                            $Result = New-Object -TypeName psobject
                            $Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
                            $Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
                            $Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress

                            $thisSourceResult = Invoke-NDKPerf1to1 -Server $thisTarget -Client $thisSource -ExpectedTPUT $Definitions.NDKPerf.TPUT

                            $Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisSourceResult.ReceiverLinkSpeedGbps
                            $Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.ReceivedGbps
                            $Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisSourceResult.ReceivedPctgOfLinkSpeed
                            $Result | Add-Member -MemberType NoteProperty -Name MinExpectedPctgOfLinkSpeed -Value $Definitions.NDKPerf.TPUT

                            if ($thisSourceResult.ReceivedPctgOfLinkSpeed -and $Definitions.NDKPerf.TPUT) {
                                if ($thisSourceResult.ReceivedPctgOfLinkSpeed -ge $Definitions.NDKPerf.TPUT) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
                                else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }
                            }
                            else {
                                $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
                                "ERROR: Data failed to be collected for path $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                            }

                            $Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData

                            Return $Result
                        })

                        $param = @{
                            thisSource  = $pair.Source
                            thisTarget  = $pair.Target
                            Definitions = $Definitions
                        }

                        [void] $PowerShell.AddParameters($param)

                        Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($pair.Source.IPAddress) -> ($($pair.Target.NodeName)) $($pair.Target.IPAddress)"
                        ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($pair.Source.IPAddress) -> ($($pair.Target.NodeName)) $($pair.Target.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                        $asyncJobObj = @{ JobHandle   = $PowerShell
                                            AsyncHandle = $PowerShell.BeginInvoke() }

                        $GroupedJobs += $asyncJobObj
                    }

                    While ($GroupedJobs -ne $null) {
                        $GroupedJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
                            $thisJob = $_
                            $StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
                            $thisReceiverHostName = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).ReceiverHostName
                            $thisSource = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Sender
                            $thisTarget = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Receiver

                            $GroupedJobs = $GroupedJobs -ne $thisJob

                            Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)"
                            ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                        }
                    }
                }

                $RunspacePool.Close()
                $RunspacePool.Dispose()
            } else { # Run sequentially
                Write-Host "Beginning Stage: $thisStage - RDMA Perf 1:1 - Sequential - $([System.DateTime]::Now)"
                "Beginning Stage: $thisStage - RDMA Perf 1:1 - Sequential - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                $NDKServerLogPath = Join-Path -Path $LogFileParentPath -ChildPath "\NDK\NDKPerfServerOutput-$(Get-Date -f yyyy-MM-dd-HHmmss).txt"
                $NDKClientLogPath = Join-Path -Path $LogFileParentPath -ChildPath "\NDK\NDKPerfClientOutput-$(Get-Date -f yyyy-MM-dd-HHmmss).txt"
                $NDKServerLog = New-Item -Path $NDKServerLogPath -ItemType File -Force -ErrorAction SilentlyContinue
                $NDKClientLog = New-Item -Path $NDKClientLogPath -ItemType File -Force -ErrorAction SilentlyContinue

                $StageResults = @()
                $TestableNetworks | ForEach-Object {
                $thisTestableNet = $_

                $thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
                    $thisSource = $_
                    $thisSourceResult = @()
                    
                    $thisTestableNet.Group | Where-Object NodeName -ne $thisSource.NodeName | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
                        $thisTarget = $_

                        $Result = New-Object -TypeName psobject
                        $Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
                        $Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
                        $Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress

                        Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
                        ":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                        $thisSourceResult = Invoke-NDKPerf1to1 -Server $thisTarget -Client $thisSource -ExpectedTPUT $Definitions.NDKPerf.TPUT -NDKServerLog $NDKServerLog -NDKClientLog $NDKClientLog

                        Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
                        ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                        $Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisSourceResult.ReceiverLinkSpeedGbps
                        $Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.ReceivedGbps
                        $Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisSourceResult.ReceivedPctgOfLinkSpeed
                        $Result | Add-Member -MemberType NoteProperty -Name MinExpectedPctgOfLinkSpeed -Value $Definitions.NDKPerf.TPUT

                        if ($thisSourceResult.ReceivedPctgOfLinkSpeed -ge $Definitions.NDKPerf.TPUT) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
                        else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }

                        $Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData

                        $StageResults += $Result
                        Remove-Variable Result -ErrorAction SilentlyContinue
                    }
                }
            }
            }

            if ('Fail' -in $StageResults.PathStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage4 -Value 'Fail'; $StageFailures++ }
            else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage4 -Value 'Pass' }

            $NetStackResults | Add-Member -MemberType NoteProperty -Name Stage4 -Value $StageResults
            Write-Host "Completed Stage: $thisStage - RDMA Perf 1:1 - $([System.DateTime]::Now)"
            "Completed Stage: $thisStage - RDMA Perf 1:1 - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Stage 4 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            $StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        }

        '5' { # RDMA Stress N:1
            if ( $ContinueOnFailure -eq $false ) {
                if ('fail' -in $NetStackResults.Stage3.PathStatus -or 'fail' -in $NetStackResults.Stage4.PathStatus) {

                    $Stage -ge 5 | ForEach-Object {
                        $AbortedStage = $_
                        $NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
                    }

                    Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
                    "Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                    return $NetStackResults
                }
            }

            $thisStage = $_
            Write-Host "Beginning Stage: $thisStage - RDMA Perf N:1 - $([System.DateTime]::Now)"
            "Stage 5`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Beginning Stage: $thisStage - RDMA Perf N:1 - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

            $StageResults = @()
            $TestableNetworks | ForEach-Object {
                $thisTestableNet = $_

                $thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
                    $thisTarget = $_
                    $ClientNetwork = @($thisTestableNet.Group | Where-Object NodeName -ne $thisTarget.NodeName | Where-Object -FilterScript { $_.RDMAEnabled })

                    Write-Host ":: $([System.DateTime]::Now) :: [Started] N -> Interface $($thisTarget.InterfaceIndex) ($($thisTarget.IPAddress))"
                    ":: $([System.DateTime]::Now) :: [Started] N -> Interface $($thisTarget.InterfaceIndex) ($($thisTarget.IPAddress))" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                    $thisTargetResult = Invoke-NDKPerfNto1 -Server $thisTarget -ClientNetwork $ClientNetwork -ExpectedTPUT $Definitions.NDKPerf.TPUT

                    $Result = New-Object -TypeName psobject
                    $Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
                    $Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress

                    $Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisTargetResult.ReceiverLinkSpeedGbps
                    $Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisTargetResult.RxGbps
                    $Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisTargetResult.ReceivedPctgOfLinkSpeed

                    if ($thisTargetResult.ServerSuccess) { $Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Pass' }
                    else { $Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Fail' }

                    $Result | Add-Member -MemberType NoteProperty -Name ClientNetworkTested -Value $thisTargetResult.ClientNetworkTested
                    $Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisTargetResult.RawData

                    $StageResults += $Result
                    Remove-Variable Result -ErrorAction SilentlyContinue

                    Write-Host ":: $([System.DateTime]::Now) :: [Completed] N -> Interface $($thisTarget.InterfaceIndex) ($($thisTarget.IPAddress))"
                    ":: $([System.DateTime]::Now) :: [Completed] N -> Interface $($thisTarget.InterfaceIndex) ($($thisTarget.IPAddress))" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                }
            }

            if ('Fail' -in $StageResults.ReceiverStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage5 -Value 'Fail'; $StageFailures++ }
            else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage5 -Value 'Pass' }

            $NetStackResults | Add-Member -MemberType NoteProperty -Name Stage5 -Value $StageResults

            Write-Host "Completed Stage: $thisStage - RDMA Perf N:1 - $([System.DateTime]::Now)"
            "Completed Stage: $thisStage - RDMA Perf N:1 - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Stage 5 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            $StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        }

        '6' { # RDMA Stress N:N
            if ( $ContinueOnFailure -eq $false ) {
                if ('fail' -in $NetStackResults.Stage3.PathStatus -or 'fail' -in $NetStackResults.Stage4.PathStatus -or 'fail' -in $NetStackResults.Stage5.PathStatus) {

                    $Stage -ge 6 | ForEach-Object {
                        $AbortedStage = $_
                        $ResultsSummary | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
                    }

                    Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
                    "Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                    return $NetStackResults
                }
            }

            $thisStage = $_
            Write-Host "Beginning Stage: $thisStage - RDMA Perf N:N - $([System.DateTime]::Now)"
            "Stage 6`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Beginning Stage: $thisStage - RDMA Perf N:N - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

            $StageResults = @()
            $TestableNetworks | ForEach-Object {
                $thisTestableNet = $_

                $ServerList = $thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled }

                $thisSubnet = ($ServerList | Select-Object -First 1).subnet
                $thisVLAN = ($ServerList | Select-Object -First 1).VLAN

                Write-Host ":: $([System.DateTime]::Now) :: [Started] N -> N on subnet $($thisSubnet) and VLAN $($thisVLAN)"
                ":: $([System.DateTime]::Now) :: [Started] N -> N on subnet $($thisSubnet) and VLAN $($thisVLAN)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                $thisSourceResult = Invoke-NDKPerfNtoN -ServerList $ServerList -ExpectedTPUT $Definitions.NDKPerf.TPUT

                $Result = New-Object -TypeName psobject
                $thisSubnet = $thisTestableNet.Name.Split(',')[0]
                $thisVLAN = $thisTestableNet.Name.Split(',')[1].Trim()
                $Result | Add-Member -MemberType NoteProperty -Name Subnet -Value $thisSubnet
                $Result | Add-Member -MemberType NoteProperty -Name VLAN -Value $thisVLAN

                $Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.RxGbps

                if ($thisSourceResult.ServerSuccess) { $Result | Add-Member -MemberType NoteProperty -Name NetworkStatus -Value 'Pass' }
                else { $Result | Add-Member -MemberType NoteProperty -Name NetworkStatus -Value 'Fail' }

                $Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData

                Write-Host ":: $([System.DateTime]::Now) :: [Completed] N -> N on subnet $($thisSubnet) and VLAN $($thisVLAN)"
                ":: $([System.DateTime]::Now) :: [Completed] N -> N on subnet $($thisSubnet) and VLAN $($thisVLAN)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

                $StageResults += $Result
                Remove-Variable Result -ErrorAction SilentlyContinue
            }

            if ('Fail' -in $StageResults.NetworkStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage6 -Value 'Fail'; $StageFailures++ }
            else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage6 -Value 'Pass' }

            $NetStackResults | Add-Member -MemberType NoteProperty -Name Stage6 -Value $StageResults

            Write-Host "Completed Stage: $thisStage - RDMA Perf N:N - $([System.DateTime]::Now)"
            "Completed Stage: $thisStage - RDMA Perf N:N - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Stage 6 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            $StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        }

        '7' { # RDMA Stress N:1 Parallel

            if ($Experimental -eq $false) {
                Write-Error "Stage $_ is experimental. The experimental flag has not been set. Please enable it to run experimental stages."
                "The experimental stage(s) $ChosenStages have been selected to be run, but the experimental flag has not been set. Please enable it to run experimental stages." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                return $NetStackResults
            }

            $IsVDiskUnhealthy = Get-VDiskStatus($LogFile)
            if ($IsVDiskUnhealthy) { 
                $Stage -ge 7 | ForEach-Object {
                    $AbortedStage = $_
                    $NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
                }

                Write-Warning 'Aborted due to unhealthy VDisk.' | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            }

            if ( $ContinueOnFailure -eq $false ) {
                if ('fail' -in $NetStackResults.Stage3.PathStatus -or 'fail' -in $NetStackResults.Stage4.PathStatus -or 'fail' -in $NetStackResults.Stage5.ReceiverStatus -or 'fail' -in $NetStackResults.Stage6.NetworkStatus) {
    
                    $Stage -ge 7 | ForEach-Object {
                        $AbortedStage = $_
                        $NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
                    }
    
                    Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
                    return $NetStackResults
                }
            }

            $NodeGroups = $Mapping | Where-Object VLAN -ne 'Unsupported' | Group-Object NodeName
            $thisStage = $_
            Write-Host "Beginning Stage: $thisStage - RDMA Perf VMSwitch Stress - $([System.DateTime]::Now)"
            "Stage 7`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
            "Beginning Stage: $thisStage - RDMA Perf VMSwitch Stress - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

            $StageResults = @()
            $ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
            $NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
            $NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }

            $NodeGroups | ForEach-Object {
                $GroupedJobs = @()            
                $RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
                $RunspacePool.Open()
                $testNodeGroup = $_  
                $testNodeGroup.Group | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
                    
                    $thisSource  = $_
                    $PowerShell = [powershell]::Create()
                    $PowerShell.RunspacePool = $RunspacePool
                    $ClientNodes = @($Mapping | Where-Object NodeName -ne $thisSource.NodeName | Where-Object VLAN -eq $thisSource.VLAN | Where-Object Subnet -eq $thisSource.Subnet | Where-Object -FilterScript { $_.RDMAEnabled })
                    
                    [void] $PowerShell.AddScript({
                        param ( $thisSource, $ClientNodes, $Definitions, $LogFile )
                        $StartTime = Get-Date
                        Write-Host ":: $([System.DateTime]::Now) :: [Started] N -> Interface $($thisSource.InterfaceIndex) ($($thisSource.IPAddress))"
                        ":: $([System.DateTime]::Now) :: [Started] N -> Interface $($thisTarget.InterfaceIndex) ($($thisSource.IPAddress))" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                        $events = @()

                        $thisSourceResult = Invoke-NDKPerfNto1 -Server $thisSource -ClientNetwork $ClientNodes -ExpectedTPUT $Definitions.NDKPerf.TPUT

                        $events = (Get-EventLog System -InstanceId 0x466,0x467,0x469,0x46a -After $StartTime)

                        $Result = New-Object -TypeName psobject
                        $Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisSource.NodeName
                        $Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisSource.IPAddress
                        $Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisSourceResult.ReceiverLinkSpeedGbps
                        $Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.RxGbps

                        if ($events) { 
                            Write-Host "Found cluster membership lost events when testing node $($thisSource.NodeName). They can be found in the test log."
                            "Found cluster membership lost events when testing node $($thisSource.NodeName). They can be found in the test log." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                            $events | Select-Object Time, EntryType, InstanceID, Message | Format-Table -AutoSize | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                        }

                        if ($thisSourceResult.ServerSuccess -and $events.count -eq 0) { $Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Pass' }
                        else { $Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Fail' }
        
                        $Result | Add-Member -MemberType NoteProperty -Name ClientNetworkTested -Value $thisSourceResult.ClientNetworkTested
                        $Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData
                        
                        Write-Host ":: $([System.DateTime]::Now) :: [Completed] N -> Interface $($thisSource.InterfaceIndex) ($($thisSource.IPAddress))"
                        ":: $([System.DateTime]::Now) :: [Completed] N -> Interface $($thisSource.InterfaceIndex) ($($thisSource.IPAddress))" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                        
                        return $Result
                    })

                    $param = @{
                        thisSource = $thisSource
                        ClientNodes = $ClientNodes
                        Definitions = $Definitions
                        LogFile = $LogFile
                    }

                    [void] $PowerShell.AddParameters($param)

                    $asyncJobObj = @{ JobHandle   = $PowerShell
                        AsyncHandle = $PowerShell.BeginInvoke() }

                    $GroupedJobs += $asyncJobObj 
                
                }

                While ($null -ne $GroupedJobs) {
                    $GroupedJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
                        $thisJob = $_
                        $StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
                        
                        $GroupedJobs = $GroupedJobs | Where-Object { $_ -ne $thisJob }
                    }
                }

                $RunspacePool.close()
                $RunspacePool.Dispose()
                
            }

            $IsVDiskUnhealthy = Get-VDiskStatus($LogFile)

            if ('Fail' -in $StageResults.ReceiverStatus -or $IsVDiskUnhealthy) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage7 -Value 'Fail'; $StageFailures++ }
                else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage7 -Value 'Pass' }
                
                $NetStackResults | Add-Member -MemberType NoteProperty -Name Stage7 -Value $StageResults
                Write-Host "Completed Stage: $thisStage - RDMA Perf VMSwitch Stress - $([System.DateTime]::Now)`r`n"
                "Completed Stage: $thisStage - RDMA Perf VMSwitch Stress - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                "Stage 7 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                $StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
                "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
        }
    }

    if ($StageFailures -gt 0) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name NetStack -Value 'Fail' }
    else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name NetStack -Value 'Pass' }

    "Net Stack Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
    $ResultsSummary | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
    "####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000

    $NetStackResults | Add-Member -MemberType NoteProperty -Name ResultsSummary -Value $ResultsSummary

    $Failures = Get-Failures -NetStackResults $NetStackResults
    if (@($Failures.PSObject.Properties).Count -gt 0) {
        $NetStackResults | Add-Member -MemberType NoteProperty -Name Failures -Value $Failures
        Write-RecommendationsToLogFile -NetStackResults $NetStackResults -LogFile $LogFile
    }
    Write-Verbose "Log file stored at: $LogPath"

    Return $NetStackResults
}