TMConsole.Client

3.0.6

Public/WebSocketClient.ps1

function Start-TMConsoleWebSocketClient {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $True)]
        [String]$WebSocketServer,

        [Parameter(Mandatory = $True)]
        [Int]$WebSocketPort,

        [Parameter()]$HostPID = -1,

        [Parameter(mandatory = $false)]
        [Bool]$OutputVerbose = $false,

        [Parameter()]
        [Bool]$AllowInsecureSSL = $False,

        [Parameter(Mandatory = $false)]
        [string]$WebSocketEncryptedAccessKey,

        [Parameter(Mandatory = $false)]
        [Version]$TMCVersion = '0.0.0'
    )

    begin {
        $global:AllowInsecureSSL = $AllowInsecureSSL
        $global:OutputVerbose = $OutputVerbose

        ## Enable Verbose Output if requested
        if ($global:OutputVerbose) {
            $global:VerbosePreference = 'Continue'
            $VerbosePreference = 'Continue'
            Write-Output 'Starting PowerShell Web Socket Client'
        }

        ## Define the Script Block that is used to wrap and invoke the ActionRequest
        $global:ScriptBlock_ActionRequestInnerJobScriptBlock = [scriptblock] {
            param(
                $ActionRequest,
                $AllowInsecureSSL,
                [EventQueue[psobject]]$TMConsoleActionRequestQueue
            )
            
            ## Ensure all streams are disabled
            $InformationPreference = 'SilentlyContinue'
            $VerbosePreference = 'SilentlyContinue'
            $ProgressPreference = 'SilentlyContinue'
            $DebugPreference = 'SilentlyContinue'
            $WarningPreference = 'SilentlyContinue'
            $Global:InformationPreference = 'SilentlyContinue'
            $Global:VerbosePreference = 'SilentlyContinue'
            $Global:ProgressPreference = 'SilentlyContinue'
            $Global:DebugPreference = 'SilentlyContinue'
            $Global:WarningPreference = 'SilentlyContinue'
            $ErrorActionPreference = 'SilentlyContinue'

            $TaskTMSession = $null
            try {
                ## Import the TMC Action Request
                Import-TMCActionRequest -PSObjectActionRequest $ActionRequest
                $TaskTMSession = $Global:TMSessions.Default

                # ## Create a Parameters Variable from the Action Script
                New-Variable -Name Params -Scope Global -Value $ActionRequest.params -Force

                ## Enable Logging
                if ($ActionRequest.logPath) {

                    ## Trim any quote characters from the Log Path
                    $RootLogPath = $ActionRequest.logPath.trim('"').trim("'")
                    Test-FolderPath -FolderPath $RootLogPath

                    ## Create the Transcript Folder Path
                    $ProjectFolder = Join-Path -Path $RootLogPath ($Global:TM.Server.Url -replace '.transitionmanager.net', '') $Global:TM.Project.Name $Global:TM.Event.Name
                    Test-FolderPath -FolderPath $ProjectFolder

                    ## Create a unique file in the Transcript Folder
                    $TranscriptFileName = (
                        (Get-Date -Format FileDateTimeUniversal) +
                        '_TaskNumber-' + $Global:TM.Task.TaskNumber +
                        '_TaskId-' + $Global:TM.Task.Id + '.txt'
                    )
                    $TranscriptFilePath = Join-Path $ProjectFolder $TranscriptFileName

                    ## Start a transcript for this session
                    $TranscriptSplat = @{
                        Path                    = $TranscriptFilePath
                        IncludeInvocationHeader = $True
                        Confirm                 = $False
                        Force                   = $True
                        Append                  = $True
                    }
                    Start-Transcript @TranscriptSplat
                }

                ## Create a Data Options parameter for the Complete-TMTask command
                $CompleteTaskParameters = @{}

                ## Add SSL Exception if necessary
                if ($global:AllowInsecureSSL) {
                    $CompleteTaskParameters | Add-Member -NotePropertyName 'AllowInsecureSSL' -NotePropertyValue $True
                }

                $InvocationError = $Null
                ## Run the User Provided Script
                $InvocationErrorText = $Null
                try {
                    ## Invoke the User Script block
                    $Stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
                    $ActionScriptBlock = [scriptblock]::Create($ActionRequest.options.apiAction.script)
                    Invoke-Command -ScriptBlock $ActionScriptBlock -ErrorAction 'Stop' -NoNewScope

                    $Stopwatch.Stop()
                    Write-Host -Message "Task completed in: " -NoNewline
                    Write-Host -Message (Get-TimeSpanString -Timespan $Stopwatch.Elapsed) -ForegroundColor Green
                }
                catch {
                    $Stopwatch.Stop()
                    $InvocationError = $_
                    $InvocationErrorText = $_?.Exception?.Message ?? $_ ?? "Error"
                }

                ## Check the Global Variable for any TMAssetUpdates to send to TransitionManager during the task completion
                if ($Global:TMAssetUpdates) {
                    $CompleteTaskParameters['Data'] = @{
                        assetUpdates = $Global:TMAssetUpdates
                    }
                    Remove-Variable -Scope Global -Name TMAssetUpdates -Force
                }

                ## End the log for this session
                if ($ActionRequest.LogPath) {
                    Stop-Transcript
                }

                if ($InvocationErrorText) {
                    Write-Host "Action Error: $InvocationErrorText" -ForegroundColor Red
                    Set-TMTaskOnHold -ActionRequest $ActionRequest -Message ('Action Error: ' + $InvocationErrorText) @CompleteTaskParameters
                    throw $InvocationError
                }
                else {
                    try {
                        Complete-TMTask -ActionRequest $ActionRequest @CompleteTaskParameters
                    }
                    catch {
                        Write-Host $_
                    }
                }
            }
            finally {
                try{
                    if ($null -ne $TaskTMSession) {
                        $TaskTMSession.Dispose()

                        if (
                            $null -ne $Global:TMSessions -and
                            $Global:TMSessions.ContainsKey('Default') -and
                            [Object]::ReferenceEquals($Global:TMSessions.Default, $TaskTMSession)
                        ) {
                            $Global:TMSessions.Remove('Default')
                        }
                    }
                }
                finally {
                    $TaskTMSession = $null
                }
            }
        }
    }

    process {
        try {
            ## Define the global TMConsole dataset
            $Global:TMConsole = @{
                
                ## TMC PS Operation Queues
                ActionRequests       = New-Object 'EventQueue[PSObject]'
                TerminateTaskActions = New-Object 'EventQueue[PSObject]'
                RemoveRunspaces      = New-Object 'TMCSessionQueue[PSObject]'
                
                ## UI Message Queues - Major events
                TaskChanges          = New-Object 'TMCSessionQueue[PSObject]'
                SystemErrors         = New-Object 'TMCSessionQueue[PSObject]'
                
                ## UI Message Queues - Aggregate events
                TaskOutput           = New-Object 'TMCSessionQueue[PSObject]'
                TaskProgress         = New-Object 'TMCSessionQueue[PSObject]'

                ## Runspaces manifest
                Runspaces            = [pscustomobject]@{}
            }
            $ActionRequestEventHandler = @{
                EventName        = 'ItemEnqueued'
                InputObject      = $Global:TMConsole.ActionRequests
                SourceIdentifier = 'TMC_ActionRequest_Enqueued'
                Action           = [scriptblock] {
                    param (
                        [Parameter()]
                        [PSObject]$ActionRequest
                    )
                    ## Rename the Variable to make the invocation logic more clear
                    $TMTaskID = 'TMTaskId_' + [string]$ActionRequest.task.id
                    $TMTaskNumber = $ActionRequest.task.taskNumber.ToString()
                    
                    ## Send the StatusMessages to TMConsole
                    $Global:TMConsole.TaskChanges.Enqueue(
                        [PSCustomObject]@{
                            TMTaskId = $TMTaskId
                            Type     = 'TaskStarted'
                        }
                    )

                    $Global:TMConsole.TaskProgress.Enqueue(
                        [PSCustomObject]@{
                            TMTaskId = $TMTaskID
                            Type     = 'Progress'
                            Message  = [PSCustomObject]@{
                                ActivityId        = 0
                                ParentActivityId  = -1
                                Activity          = 'Starting Task: ' + $TMTaskNumber + ' - ' + $ActionRequest.task.title
                                CurrentOperation  = 'Starting'
                                StatusDescription = ''
                                PercentComplete   = 0
                                SecondsRemaining  = -1
                                RecordType        = 0
                            }
                        }    
                    )

                    ##
                    ## Create the Runspace
                    ##
                    $Powershell = [powershell]::Create()
                    $Powershell.RunspacePool = $Global:RunspacePool
                    
                    ## Add a $TMTaskId variable so the Pipeline knows who the output belongs to
                    [void]($Powershell.AddScript(('$TMTaskId = "{0}"' -f $TMTaskID)))
                    [void]($PowerShell.AddScript({ [TMCPSHost]::CurrentTMTaskId.Value = $ExecutionContext.SessionState.PSVariable.GetValue("TMTaskId") }))

                    [void]($Powershell.AddScript($global:ScriptBlock_ActionRequestInnerJobScriptBlock))
                    [void]($Powershell.AddArgument($ActionRequest))
                    [void]($Powershell.AddArgument($global:AllowInsecureSSL))
                    [void]($Powershell.AddArgument($global:TMConsole.ActionRequests))
                    
                    ## Start the Runspace and record the Async handle
                    try {
                        $AddMemberSplat = @{
                            InputObject      = $Global:TMConsole.Runspaces 
                            NotePropertyName = $TMTaskID 
                            Force            = $True
                        }
                        [void](Add-Member @AddMemberSplat -NotePropertyValue @{
                                TMTaskId     = $TMTaskId
                                TMTaskNumber = $TMTaskNumber
                                ErrorText    = ''
                                Runspace     = $Powershell 
                                AsyncHandle  = $Powershell.BeginInvoke()
                            }
                        )
                        [TMCRunspaceDirectory]::ActiveRunspaces[$TMTaskID] = 'Running'
                    }
                    catch {
                        Write-Host -ForegroundColor "Red" "ScriptBlock_TaskRunspace_InvokeActionRequest - Invocation Error $_"
                        $Global:TMConsole.SystemErrors.Enqueue(
                            [PSCustomObject]@{
                                Type      = 'SystemError'
                                From      = 'Starting ActionRequest RunspaceJob'
                                Message   = 'ScriptBlock_TaskRunspace_InvokeActionRequest - Invocation Error'
                                Exception = $_.Exception.Message
                            }
                        )
                    }

                }
            }
            [void](Register-ObjectEvent @ActionRequestEventHandler)

            $TerminateTaskActionEventHandler = @{
                EventName        = 'ItemEnqueued'
                InputObject      = $Global:TMConsole.TerminateTaskActions
                SourceIdentifier = 'TMC_TerminateTaskAction_Enqueued'
                Action           = [scriptblock] {
                    param($TMTask)
                    $TMTaskId = "TMTaskId_$($TMTask.task.id)"
                    try {

                        ## Send Terminated task output to the console in TMC
                        $Global:TMConsole.TaskProgress.Enqueue([PSCustomObject]@{
                                TMTaskId = $TMTaskId
                                Type     = 'Progress'
                                Message  = [PSCustomObject]@{
                                    ActivityId        = 0
                                    ParentActivityId  = -1
                                    Activity          = 'Task Terminated'
                                    CurrentOperation  = 'Terminated!'
                                    StatusDescription = ''
                                    PercentComplete   = -1
                                    SecondsRemaining  = -1
                                    RecordType        = 1
                                }
                            }
                        )
                        $Global:TMConsole.TaskOutput.Enqueue(
                            ## Force a line break
                            [PSCustomObject]@{
                                TMTaskId = $TMTaskId
                                Type     = 'Information'
                                Message  = @{
                                    Message         = ""
                                    NoNewLine       = $False
                                    ForegroundColor = 'White'
                                    BackgroundColor = 'Black'
                                }
                            }
                        )
                        $Global:TMConsole.TaskOutput.Enqueue(
                            ## Print a message that the task was terminated
                            [PSCustomObject]@{
                                TMTaskId = $TMTaskId
                                Type     = 'Information'
                                Message  = @{
                                    Message         = "Task has been terminated!"
                                    NoNewLine       = $False
                                    ForegroundColor = "Red"
                                    BackgroundColor = "Black"
                                }
                            }
                        )
                        try {
                            ## Request an early, non-blocking stop. Disposal is handled by the
                            ## RemoveRunspaces drain in the main loop.
                            [void]($Global:TMConsole.Runspaces.$TMTaskId.Runspace.BeginStop($Null, $Null))
                        }
                        catch { }
                        $Global:TMConsole.RemoveRunspaces.Enqueue($TMTaskId)
                    }
                    catch {
                        Write-Host -ForegroundColor "Red" "RSJob_TerminateTaskAction FAILED removing the RSJob $_"
                        $Global:TMConsole.SystemErrors.Enqueue(
                            [PSCustomObject]@{
                                Type   = 'SystemError'
                                From   = 'Terminating Runspace'
                                Detail = 'RSJob_TerminateTaskAction FAILED removing the RSJob'
                                Error  = $_.Exception.Message
                            }
                        )
                    }
                }
            }
            [void](Register-ObjectEvent @TerminateTaskActionEventHandler)
           
            ##
            ## Prepare Runspaces for execution
            ##
            $InitialSessionState = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
            $InitialSessionState.ImportPSModule('TMD.Common')
            $InitialSessionState.ImportPSModule('TransitionManager')
            $TMCPsHost = New-Object -TypeName TMCPsHost
            $MaxRunspaces = [Math]::Min([Environment]::ProcessorCount * 8, 64)
            $global:RunspacePool = [runspacefactory]::CreateRunspacePool(
                1, 
                $MaxRunspaces,
                $InitialSessionState,
                $TMCPsHost
            )
            $global:RunspacePool.ApartmentState = "MTA"
            $global:RunspacePool.Open()

            ##
            ## Build the Web Socket Client and connect
            ##
            $global:CancellationToken = New-Object System.Threading.CancellationToken
            $global:WebSocketClient = [WatsonWebsocket.WatsonWsClient]::new($WebSocketServer, $WebSocketPort, $true)
            $WebSocketReceiveMessageEventHandler = @{
                InputObject      = $global:WebSocketClient
                EventName        = 'MessageReceived'
                SourceIdentifier = 'TMC_Websocket_MessageReceived'
                Action           = [scriptblock] {
                    try {

                        ## WebSocket text messages are UTF-8 encoded byte arrays. Decode the
                        ## complete payload at once instead of rebuilding it one byte at a time.
                        $Message = ConvertFrom-Json -InputObject (
                            [System.Text.Encoding]::UTF8.GetString($Event.SourceEventArgs.Data)
                        )

                        ## Queue the Message in the appropriate Queue
                        switch ($Message.Type) {
                            'ActionRequest' {
                                ## Run the Action Request
                                $Global:TMConsole.ActionRequests.Enqueue($Message)
                                break
                            }
                            'TerminateTaskAction' {
                                ## Run the Action Request
                                $Global:TMConsole.TerminateTaskActions.Enqueue($Message)
                                break
                            }
                        }
                    }
                    catch {
                        Write-Host -ForegroundColor "Red" "EventHandler_WebSocket_ReceiveData $_"
                        $Global:TMConsole.SystemErrors.Enqueue(
                            [PSCustomObject]@{
                                Type  = 'Debug THROWN ERROR'
                                Error = $_
                            }
                        )
                    }
                }
            }
            [void](Register-ObjectEvent @WebSocketReceiveMessageEventHandler)

            ##
            ## Establish the WebSocket connection
            ##
            try {
                Write-Verbose $WebSocketEncryptedAccessKey
                $Cookie = New-Object System.Net.Cookie
                $Cookie.Name = 'tm-access-key'
                $Cookie.Value = $WebSocketEncryptedAccessKey
                $Cookie.Domain = $WebSocketServer
                $global:WebSocketClient.AddCookie($Cookie)
                $global:WebSocketClient.Start()
            }
            catch {
                Write-Host -ForegroundColor "Red" "Connection Faulted: $($_.Exception.Message)"
                throw "Connection Faulted: $($_.Exception.Message)"
            }
            
            if (-not $global:WebSocketClient.Connected) {
                Write-Host -ForegroundColor "Red" "PowerShell was unable to connect to TMConsole WebSocketServer at $($WebSocketServer):$($WebSocketPort)"
                throw "PowerShell was unable to connect to TMConsole WebSocketServer at $($WebSocketServer):$($WebSocketPort)"
            }
            
            ## Warm up each runspace so the modules load
            1..$MaxRunspaces | ForEach-Object {
                $PrepShell = [powershell]::Create()
                $PrepShell.RunspacePool = $Global:RunspacePool
                $Null = $PrepShell.AddScript('').Invoke()
            }
            
            ##
            ## Perform primary loop,
            ## Receives ActionRequets and TerminateTask items from Angular
            ## Sends UI update messages to Angular
            ## Manages RunspacePool
            ##
            $LastBatchSend = [datetime]::UtcNow
            $LastRunningActionCount = -1
            $MinBatchSize = 5
            $MaxBatchSize = 50
            $MaxBatchLatency = 50
            $SleepInterval = 10
            $MessageBatch = [System.Collections.Generic.List[object]]@()

            do {
                
                ## Check for any completed runspaces
                foreach ($TMTaskID in $Global:TMConsole.Runspaces.PSObject.Properties.Name) {
                    $TMRunspace = $Global:TMConsole.Runspaces.$TMTaskID
                    if ($TMRunspace.AsyncHandle.IsCompleted) {
                        $InvocationStateInfo = $TMRunspace.Runspace.InvocationStateInfo
                        $InvocationErrorRecord = $TMRunspace.Runspace.Streams.Error | Select-Object -Last 1

                        ## Preserve HadErrors as the primary compatibility signal. The custom
                        ## PSHost can report an error without adding an ErrorRecord to
                        ## Streams.Error. The Failed state also identifies uncaught terminating
                        ## errors surfaced directly by the PowerShell hosting API.
                        $InvocationFailed = (
                            $TMRunspace.Runspace.HadErrors -or
                            $InvocationStateInfo.State -eq [System.Management.Automation.PSInvocationState]::Failed
                        )

                        if ($InvocationFailed) {
                            ## Uncaught terminating errors may not be added to Streams.Error by
                            ## the hosting API, but their exception is retained as State.Reason.
                            $InvocationError = $InvocationStateInfo.Reason?.Message

                            if ([String]::IsNullOrWhiteSpace($InvocationError) -and $null -ne $InvocationErrorRecord) {
                                $InvocationError = $InvocationErrorRecord.Exception?.Message
                                if ([String]::IsNullOrWhiteSpace($InvocationError)) {
                                    $InvocationError = $InvocationErrorRecord.ToString()
                                }
                            }

                            ## Write-Host is carried on the Information stream. Keep it as a
                            ## compatibility fallback for errors rendered by the custom PSHost.
                            if (
                                [String]::IsNullOrWhiteSpace($InvocationError) -and
                                $TMRunspace.Runspace.Streams.Information.Count -gt 0
                            ) {
                                $LastInformationData = $TMRunspace.Runspace.Streams.Information[-1].MessageData
                                $InvocationError = $LastInformationData?.Message
                                if ([String]::IsNullOrWhiteSpace($InvocationError) -and $null -ne $LastInformationData) {
                                    $InvocationError = $LastInformationData.ToString()
                                }
                            }

                            if ([String]::IsNullOrWhiteSpace($InvocationError)) {
                                $InvocationError = 'The task failed without providing error details.'
                            }

                            ## Send the StatusMessages to TMConsole
                            $Global:TMConsole.TaskProgress.Enqueue([PSCustomObject]@{
                                    TMTaskId = $TMTaskId
                                    Type     = 'Progress'
                                    Message  = [PSCustomObject]@{
                                        ActivityId        = 0
                                        ParentActivityId  = -1
                                        Activity          = 'Task Failed'
                                        CurrentOperation  = 'Failed'
                                        StatusDescription = ''
                                        PercentComplete   = 100
                                        SecondsRemaining  = -1
                                        RecordType        = 2
                                    }
                                }
                            )
                            $Global:TMConsole.TaskChanges.Enqueue(
                                [PSCustomObject]@{
                                    TMTaskId     = $TMTaskId
                                    TMTaskNumber = $TMRunspace.TMTaskNumber
                                    Type         = 'Error'
                                    Message      = $InvocationError
                                }
                            )
                            $Global:TMConsole.TaskChanges.Enqueue(
                                [PSCustomObject]@{
                                    TMTaskId     = $TMTaskId
                                    TMTaskNumber = $TMRunspace.TMTaskNumber
                                    Type         = 'TaskFailed'
                                    Message      = $InvocationError
                                }
                            )
                        }
                        else {
                            $Global:TMConsole.TaskProgress.Enqueue( 
                                [PSCustomObject]@{
                                    TMTaskId = $TMTaskID
                                    Type     = 'Progress'
                                    Message  = [PSCustomObject]@{
                                        ActivityId        = 0
                                        ParentActivityId  = -1
                                        Activity          = 'Task Completed'
                                        CurrentOperation  = 'Complete'
                                        StatusDescription = ''
                                        PercentComplete   = 100
                                        SecondsRemaining  = -1
                                        RecordType        = 1
                                    }
                                }
                            )
                            $Global:TMConsole.TaskChanges.Enqueue( 
                                [PSCustomObject]@{
                                    TMTaskId = $TMTaskId
                                    Type     = 'TaskCompleted'
                                }
                            )
                        }
                        $Global:TMConsole.RemoveRunspaces.Enqueue($TMTaskId)
                    }
                }

                ## Collect all of the Task State Change tokens
                while ($Global:TMConsole.RemoveRunspaces.Count) {
                    $TMTaskId = $null
                    if ($Global:TMConsole.RemoveRunspaces.TryDequeue([ref]$TMTaskId)) {

                        ## Resolve the manifest entry that belongs to the Task Id that was just
                        ## dequeued. A Task can be enqueued for removal more than once (normal
                        ## completion and an explicit termination), so this is legitimately $null
                        ## on the second pass and there is nothing left to dispose.
                        $TMRunspace = $Global:TMConsole.Runspaces.$TMTaskId

                        if ($TMRunspace -and $TMRunspace.Runspace) {

                            ## Hold the PowerShell instance locally so every call below acts on
                            ## the Task being retired and not on whatever was last enumerated
                            $TaskPowerShell = $TMRunspace.Runspace

                            ## Close out the invocation before disposing. EndInvoke re-throws a
                            ## terminating error raised by the Action script, which has already
                            ## been reported to TMConsole, so it is discarded here.
                            try {
                                if ($TMRunspace.AsyncHandle) {
                                    if ($TMRunspace.AsyncHandle.IsCompleted) {
                                        [void]($TaskPowerShell.EndInvoke($TMRunspace.AsyncHandle))
                                    }
                                    else {
                                        ## Termination path - BeginStop was already requested by the
                                        ## TerminateTaskAction handler, so this returns promptly
                                        $TaskPowerShell.Stop()
                                    }
                                }
                            }
                            catch {
                                ## Errors are present in this output but are not ours to handle here
                            }

                            ## Release the buffered stream data this Task produced
                            try {
                                [void]($TaskPowerShell.Streams.Information.Clear())
                                [void]($TaskPowerShell.Streams.Error.Clear())
                                [void]($TaskPowerShell.Streams.Warning.Clear())
                                [void]($TaskPowerShell.Streams.Verbose.Clear())
                                [void]($TaskPowerShell.Streams.Debug.Clear())
                                [void]($TaskPowerShell.Streams.Progress.Clear())
                            }
                            catch {
                                ## Errors are present in this output but are not ours to handle here
                            }

                            ## Dispose releases the pipeline and returns the runspace to the pool
                            try {
                                [void]($TaskPowerShell.Dispose())
                            }
                            catch {
                                ## Errors are present in this output but are not ours to handle here
                            }

                            $TaskPowerShell = $null
                            $TMRunspace.Runspace = $null
                            $TMRunspace.AsyncHandle = $null
                        }

                        if ($Global:TMConsole.Runspaces.PSObject.Properties.Name -contains $TMTaskId) {
                            $Global:TMConsole.Runspaces.PSObject.Properties.Remove($TMTaskId)
                        }
                        if ([TMCRunspaceDirectory]::ActiveRunspaces.Keys -contains $TMTaskId) {
                            [void]([TMCRunspaceDirectory]::ActiveRunspaces.TryRemove($TMTaskId, [ref]$null))
                        }
                    }
                }

                ## Check if the runspace count has changed
                if ($LastRunningActionCount -ne $Global:TMConsole.Runspaces.PSObject.Properties.Name.count) {
                    $LastRunningActionCount = $Global:TMConsole.Runspaces.PSObject.Properties.Name.count
                    $ActionCountLabel = switch ($LastRunningActionCount) {
                        0 { '0 Actions Running' }
                        1 { '1 Action Running' }
                        Default { "$([int]$LastRunningActionCount) Actions Running" }
                    }   
                    [void]($MessageBatch.Add([PSCustomObject]@{
                                Type    = 'powershell-server-status'
                                Message = @{
                                    connectionStatus = 'Connected'
                                    serverName       = $WebSocketServer
                                    serverStatus     = $ActionCountLabel
                                }
                            }
                        )
                    )
                }
                
                ##
                ## Priority Messages, always get sent when available
                ##
              
                ## Collect all of the Task State Change tokens
                while ($Global:TMConsole.TaskChanges.Count) {
                    $message = $null
                    if ($Global:TMConsole.TaskChanges.TryDequeue([ref]$message)) {
                        [void]($MessageBatch.Add($message))
                    }
                }
                
                ## Collect all of the System Error tokens
                while ($Global:TMConsole.SystemErrors.Count) {
                    $message = $null
                    if ($Global:TMConsole.SystemErrors.TryDequeue([ref]$message)) {
                        [void]($MessageBatch.Add($message))
                        Write-Warning $Message
                    }
                }

                if ($MessageBatch.Count -gt 0) {
                    [void]($global:WebSocketClient.SendAsync(
                            ($MessageBatch | ConvertTo-Json -EnumsAsStrings -Compress), 
                            [System.Net.WebSockets.WebSocketMessageType]::Text, 
                            $global:CancellationToken
                        )
                    )
                    [void]($MessageBatch.Clear())
                }

                ##
                ## Aggregate Console Messages and load in blocks
                ##
                $Now = [datetime]::UtcNow
                $Elapsed = ($Now - $LastBatchSend).TotalMilliseconds

                ## Collect ConsoleOutput (Fabricated ones from Pipeline, needs updating)
                if ($Global:TMConsole.TaskOutput.count -gt $MinBatchSize -or $Elapsed -gt $MaxBatchLatency) {
                    for ($i = 0; $i -lt $MinBatchSize; $i++) {
                        $message = $null
                        if ($Global:TMConsole.TaskOutput.TryDequeue([ref]$message)) {
                            # Write-Host ($message | ConvertTo-Json -EnumsAsStrings -Compress) -Foreground Yellow
                            [void]($MessageBatch.Add($message))
                        }
                    }
                }
                
                ## Collect Progress Output
                if (
                    ($MessageBatch.Count -gt 0) -or    
                    ($Global:TMConsole.TaskProgress.count -gt $MinBatchSize) -or 
                    ($Elapsed -gt $MaxBatchLatency)
                ) {
                    while ($Global:TMConsole.TaskProgress.Count) {
                        $message = $null
                        if ($Global:TMConsole.TaskProgress.TryDequeue([ref]$message)) {
                            [void]($MessageBatch.Add($message))
                        }
                    }
                }

                if ($MessageBatch.Count -gt 0) {
                    [void]($global:WebSocketClient.SendAsync(
                            ($MessageBatch | ConvertTo-Json -EnumsAsStrings -Compress), 
                            [System.Net.WebSockets.WebSocketMessageType]::Text, 
                            $global:CancellationToken
                        )
                    )
                    [void]($MessageBatch.Clear())
                    $LastBatchSend = $Now
                }

                ## TMC TMCPSHost Collection
                $message = $null
                while ([TMCPSHost]::OutputQueue.TryDequeue([ref]$message)) {
                    [void]($MessageBatch.Add($message))
                    if ($MessageBatch.Count -gt $MaxBatchSize) { break }
                }
                    
                if ($MessageBatch.Count -gt 0) {
                    [void]($global:WebSocketClient.SendAsync(
                            ($MessageBatch | ConvertTo-Json -EnumsAsStrings -Compress), 
                            [System.Net.WebSockets.WebSocketMessageType]::Text, 
                            $global:CancellationToken
                        )
                    )
                    [void]($MessageBatch.Clear())
                    $LastBatchSend = $Now
                }

                Start-Sleep -Milliseconds $SleepInterval

            } until (-Not $global:WebSocketClient.Connected)
        }
        catch {
            Write-Host -ForegroundColor "Red" "FAILURE IN WSS Client"
            throw $_
        }
        finally {
            
            $EventSubscribers = Get-EventSubscriber -SourceIdentifier "TM*" | Foreach-Object
            foreach ($TMEventSubscriber in $EventSubscribers) {
                Unregister-Event -Force -SubscriptionId $TMEventSubscriber.SubscriptionId
                $TMEventSubscriber.Action.Dispose()
                $TMEventSubscriber = $null
            }
            
            ## Disconnect and close down
            if ($global:WebSocketClient) {
                if ($global:WebSocketClient.Connected) {
                    $global:WebSocketClient.Stop()
                }
                $global:WebSocketClient.Dispose()
            }
        }
    }
}