TMConsole.Client

3.0.2

Public/WebSocketClient.ps1

function Start-TMConsoleWebSocketClient {
    [CmdletBinding()]
    param (
        [Parameter(Mandatory = $True)]
        [String]$WebSocketServer,

        [Parameter(Mandatory = $True)]
        [Int]$WebSocketPort,

        [Parameter()]$HostPID = -1,

        [Parameter(mandatory = $false)]
        [Bool]$OutputVerbose = $false,

        [Parameter()]
        [Bool]$AllowInsecureSSL = $False,

        [Parameter(Mandatory = $false)]
        [string]$WebSocketEncryptedAccessKey,

        [Parameter(Mandatory = $false)]
        [Version]$TMCVersion = '0.0.0'
    )

    begin {
        $global:AllowInsecureSSL = $AllowInsecureSSL
        $global:OutputVerbose = $OutputVerbose

        ## Enable Verbose Output if requested
        if ($global:OutputVerbose) {
            $global:VerbosePreference = 'Continue'
            $VerbosePreference = 'Continue'
            Write-Output 'Starting PowerShell Web Socket Client'
        }

        ## Define the Script Block that is used to wrap and invoke the ActionRequest
        $global:ScriptBlock_ActionRequestInnerJobScriptBlock = [scriptblock] {
            param(
                $ActionRequest,
                $AllowInsecureSSL,
                [EventQueue[psobject]]$TMConsoleActionRequestQueue
            )
            
            ## Ensure all streams are disabled
            $InformationPreference = 'SilentlyContinue'
            $VerbosePreference = 'SilentlyContinue'
            $ProgressPreference = 'SilentlyContinue'
            $DebugPreference = 'SilentlyContinue'
            $WarningPreference = 'SilentlyContinue'
            $Global:InformationPreference = 'SilentlyContinue'
            $Global:VerbosePreference = 'SilentlyContinue'
            $Global:ProgressPreference = 'SilentlyContinue'
            $Global:DebugPreference = 'SilentlyContinue'
            $Global:WarningPreference = 'SilentlyContinue'
            $ErrorActionPreference = 'SilentlyContinue'
            
            ## Import the TMC Action Request
            . Import-TMCActionRequest -PSObjectActionRequest $ActionRequest
            
            # ## Create a Parameters Variable from the Action Script
            New-Variable -Name Params -Scope Global -Value $ActionRequest.params -Force

            ## Enable Logging
            if ($ActionRequest.logPath) {

                ## Trim any quote characters from the Log Path
                $RootLogPath = $ActionRequest.logPath.trim('"').trim("'")
                Test-FolderPath -FolderPath $RootLogPath

                ## Create the Transcript Folder Path
                $ProjectFolder = Join-Path -Path $RootLogPath ($Global:TM.Server.Url -replace '.transitionmanager.net', '') $Global:TM.Project.Name $Global:TM.Event.Name
                Test-FolderPath -FolderPath $ProjectFolder

                ## Create a unique file in the Transcript Folder
                $TranscriptFileName = (
                    (Get-Date -Format FileDateTimeUniversal) +
                    '_TaskNumber-' + $Global:TM.Task.TaskNumber +
                    '_TaskId-' + $Global:TM.Task.Id + '.txt'
                )
                $TranscriptFilePath = Join-Path $ProjectFolder $TranscriptFileName

                ## Start a transcript for this session
                $TranscriptSplat = @{
                    Path                    = $TranscriptFilePath
                    IncludeInvocationHeader = $True
                    Confirm                 = $False
                    Force                   = $True
                    Append                  = $True
                }
                Start-Transcript @TranscriptSplat
            }

            ## Create a Data Options parameter for the Complete-TMTask command
            $CompleteTaskParameters = @{}
            
            ## Add SSL Exception if necessary
            if ($global:AllowInsecureSSL) {
                $CompleteTaskParameters | Add-Member -NotePropertyName 'AllowInsecureSSL' -NotePropertyValue $True
            }

            $InvocationError = $Null
            $InvocationErrorText = $Null
            ## Run the User Provided Script
            try {
                ## Invoke the User Script block
                $Stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
                $ActionScriptBlock = [scriptblock]::Create($ActionRequest.options.apiAction.script)
                Invoke-Command -ScriptBlock $ActionScriptBlock -ErrorAction 'Stop' -NoNewScope

                $Stopwatch.Stop()
                Write-Host -Message "Task completed in: " -NoNewline
                Write-Host -Message (Get-TimeSpanString -Timespan $Stopwatch.Elapsed) -ForegroundColor Green
            }
            catch {
                $Stopwatch.Stop()
                $InvocationError = $_
                $InvocationErrorText = $_?.Exception?.Message ?? $_ ?? "Error"
            }
            
            ## Check the Global Variable for any TMAssetUpdates to send to TransitionManager during the task completion
            if ($Global:TMAssetUpdates) {
                $CompleteTaskParameters['Data'] = @{
                    assetUpdates = $Global:TMAssetUpdates
                }
            }
           
            ## End the log for this session
            if ($ActionRequest.LogPath) {
                Stop-Transcript
            }
            
            if ($InvocationErrorText) {
                Write-Host "Action Error: $InvocationErrorText" -ForegroundColor Red 
                Set-TMTaskOnHold -ActionRequest $ActionRequest -Message ('Action Error: ' + $InvocationErrorText) @CompleteTaskParameters
                throw $InvocationError
            }
            else {
                Complete-TMTask -ActionRequest $ActionRequest @CompleteTaskParameters
            }
        }
    }

    process {
        try {
            ## Define the global TMConsole dataset
            $Global:TMConsole = @{
                
                ## TMC PS Operation Queues
                ActionRequests       = New-Object 'EventQueue[PSObject]'
                TerminateTaskActions = New-Object 'EventQueue[PSObject]'
                RemoveRunspaces      = New-Object 'TMCSessionQueue[PSObject]'
                
                ## UI Message Queues - Major events
                TaskChanges          = New-Object 'TMCSessionQueue[PSObject]'
                SystemErrors         = New-Object 'TMCSessionQueue[PSObject]'
                
                ## UI Message Queues - Aggregate events
                TaskOutput           = New-Object 'TMCSessionQueue[PSObject]'
                TaskProgress         = New-Object 'TMCSessionQueue[PSObject]'

                ## Runspaces manifest
                Runspaces            = [pscustomobject]@{}
            }
            $ActionRequestEventHandler = @{
                EventName        = 'ItemEnqueued'
                InputObject      = $Global:TMConsole.ActionRequests
                SourceIdentifier = 'TMC_ActionRequest_Enqueued'
                Action           = [scriptblock] {
                    param (
                        [Parameter()]
                        [PSObject]$ActionRequest
                    )
                    ## Rename the Variable to make the invocation logic more clear
                    $TMTaskID = 'TMTaskId_' + [string]$ActionRequest.task.id
                    $TMTaskNumber = $ActionRequest.task.taskNumber.ToString()
                    
                    ## Send the StatusMessages to TMConsole
                    $Global:TMConsole.TaskChanges.Enqueue(
                        [PSCustomObject]@{
                            TMTaskId = $TMTaskId
                            Type     = 'TaskStarted'
                        }
                    )

                    $Global:TMConsole.TaskProgress.Enqueue(
                        [PSCustomObject]@{
                            TMTaskId = $TMTaskID
                            Type     = 'Progress'
                            Message  = [PSCustomObject]@{
                                ActivityId        = 0
                                ParentActivityId  = -1
                                Activity          = 'Starting Task: ' + $TMTaskNumber + ' - ' + $ActionRequest.task.title
                                CurrentOperation  = 'Starting'
                                StatusDescription = ''
                                PercentComplete   = 0
                                SecondsRemaining  = -1
                                RecordType        = 0
                            }
                        }    
                    )

                    ##
                    ## Create the Runspace
                    ##
                    $Powershell = [powershell]::Create()
                    $Powershell.RunspacePool = $Global:RunspacePool
                    
                    ## Add a $TMTaskId variable so the Pipeline knows who the output belongs to
                    [void]($Powershell.AddScript(('$TMTaskId = "{0}"' -f $TMTaskID)))
                    [void]($PowerShell.AddScript({ [TMCPSHost]::CurrentTMTaskId.Value = $ExecutionContext.SessionState.PSVariable.GetValue("TMTaskId") }))

                    [void]($Powershell.AddScript($global:ScriptBlock_ActionRequestInnerJobScriptBlock))
                    [void]($Powershell.AddArgument($ActionRequest))
                    [void]($Powershell.AddArgument($global:AllowInsecureSSL))
                    [void]($Powershell.AddArgument($global:TMConsole.ActionRequests))
                    
                    ## Start the Runspace and record the Async handle
                    try {
                        $AddMemberSplat = @{
                            InputObject      = $Global:TMConsole.Runspaces 
                            NotePropertyName = $TMTaskID 
                            Force            = $True
                        }
                        [void](Add-Member @AddMemberSplat -NotePropertyValue @{
                                TMTaskId     = $TMTaskId
                                TMTaskNumber = $TMTaskNumber
                                ErrorText    = ''
                                Runspace     = $Powershell 
                                AsyncHandle  = $Powershell.BeginInvoke()
                            }
                        )
                        [TMCRunspaceDirectory]::ActiveRunspaces[$TMTaskID] = 'Running'
                    }
                    catch {
                        Write-Host -ForegroundColor "Red" "ScriptBlock_TaskRunspace_InvokeActionRequest - Invocation Error $_"
                        $Global:TMConsole.SystemErrors.Enqueue(
                            [PSCustomObject]@{
                                Type      = 'SystemError'
                                From      = 'Starting ActionRequest RunspaceJob'
                                Message   = 'ScriptBlock_TaskRunspace_InvokeActionRequest - Invocation Error'
                                Exception = $_.Exception.Message
                            }
                        )
                    }

                }
            }
            [void](Register-ObjectEvent @ActionRequestEventHandler)

            $TerminateTaskActionEventHandler = @{
                EventName        = 'ItemEnqueued'
                InputObject      = $Global:TMConsole.TerminateTaskActions
                SourceIdentifier = 'TMC_TerminateTaskAction_Enqueued'
                Action           = [scriptblock] {
                    param($TMTask)
                    $TMTaskId = "TMTaskId_$($TMTask.task.id)"
                    try {

                        ## Send Terminated task output to the console in TMC
                        $Global:TMConsole.TaskProgress.Enqueue([PSCustomObject]@{
                                TMTaskId = $TMTaskId
                                Type     = 'Progress'
                                Message  = [PSCustomObject]@{
                                    ActivityId        = 0
                                    ParentActivityId  = -1
                                    Activity          = 'Task Terminated'
                                    CurrentOperation  = 'Terminated!'
                                    StatusDescription = ''
                                    PercentComplete   = -1
                                    SecondsRemaining  = -1
                                    RecordType        = 1
                                }
                            }
                        )
                        $Global:TMConsole.TaskOutput.Enqueue(
                            ## Force a line break
                            [PSCustomObject]@{
                                TMTaskId = $TMTaskId
                                Type     = 'Information'
                                Message  = @{
                                    Message         = ""
                                    NoNewLine       = $False
                                    ForegroundColor = 'White'
                                    BackgroundColor = 'Black'
                                }
                            }
                        )
                        $Global:TMConsole.TaskOutput.Enqueue(
                            ## Print a message that the task was terminated
                            [PSCustomObject]@{
                                TMTaskId = $TMTaskId
                                Type     = 'Information'
                                Message  = @{
                                    Message         = "Task has been terminated!"
                                    NoNewLine       = $False
                                    ForegroundColor = "Red"
                                    BackgroundColor = "Black"
                                }
                            }
                        )
                        try {
                            $Global:TMConsole.Runspaces.$TMTaskId.Runspace.BeginStop(
                                $Null, 
                                $Global:TMConsole.Runspaces.$TMTaskId.Runspace.AsyncHandle
                            )
                        }
                        catch { }
                        $Global:TMConsole.RemoveRunspaces.Enqueue($TMTaskId)
                    }
                    catch {
                        Write-Host -ForegroundColor "Red" "RSJob_TerminateTaskAction FAILED removing the RSJob $_"
                        $Global:TMConsole.SystemErrors.Enqueue(
                            [PSCustomObject]@{
                                Type   = 'SystemError'
                                From   = 'Terminating Runspace'
                                Detail = 'RSJob_TerminateTaskAction FAILED removing the RSJob'
                                Error  = $_.Exception.Message
                            }
                        )
                    }
                }
            }
            [void](Register-ObjectEvent @TerminateTaskActionEventHandler)
           
            ##
            ## Prepare Runspaces for execution
            ##
            $InitialSessionState = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
            $InitialSessionState.ImportPSModule('TMD.Common')
            $InitialSessionState.ImportPSModule('TransitionManager')
            $TMCPsHost = New-Object -TypeName TMCPsHost
            $MaxRunspaces = [Math]::Min([Environment]::ProcessorCount * 8, 64)
            $global:RunspacePool = [runspacefactory]::CreateRunspacePool(
                1, 
                $MaxRunspaces,
                $InitialSessionState,
                $TMCPsHost
            )
            $global:RunspacePool.ApartmentState = "MTA"
            $global:RunspacePool.Open()

            ##
            ## Build the Web Socket Client and connect
            ##
            $global:CancellationToken = New-Object System.Threading.CancellationToken
            $global:WebSocketClient = [WatsonWebsocket.WatsonWsClient]::new($WebSocketServer, $WebSocketPort, $true)
            $WebSocketReceiveMessageEventHandler = @{
                InputObject      = $global:WebSocketClient
                EventName        = 'MessageReceived'
                SourceIdentifier = 'TMC_Websocket_MessageReceived'
                Action           = [scriptblock] {
                    try {

                        ## Create an Event to receive the next Character Array
                        $MessageString = ''
                        $Event.SourceEventArgs.Data | ForEach-Object {
                            $MessageString += [char]$_
                        }
                        $Message = $MessageString | ConvertFrom-Json

                        ## Queue the Message in the appropriate Queue
                        switch ($Message.Type) {
                            'ActionRequest' {
                                ## Run the Action Request
                                $Global:TMConsole.ActionRequests.Enqueue($Message)
                                break
                            }
                            'TerminateTaskAction' {
                                ## Run the Action Request
                                $Global:TMConsole.TerminateTaskActions.Enqueue($Message)
                                break
                            }
                        }
                    }
                    catch {
                        Write-Host -ForegroundColor "Red" "EventHandler_WebSocket_ReceiveData $_"
                        $Global:TMConsole.SystemErrors.Enqueue(
                            [PSCustomObject]@{
                                Type  = 'Debug THROWN ERROR'
                                Error = $_
                            }
                        )
                    }
                }
            }
            [void](Register-ObjectEvent @WebSocketReceiveMessageEventHandler)

            ##
            ## Establish the WebSocket connection
            ##
            try {
                Write-Verbose $WebSocketEncryptedAccessKey
                $Cookie = New-Object System.Net.Cookie
                $Cookie.Name = 'tm-access-key'
                $Cookie.Value = $WebSocketEncryptedAccessKey
                $Cookie.Domain = $WebSocketServer
                $global:WebSocketClient.AddCookie($Cookie)
                $global:WebSocketClient.Start()
            }
            catch {
                Write-Host -ForegroundColor "Red" "Connection Faulted: $($_.Exception.Message)"
                throw "Connection Faulted: $($_.Exception.Message)"
            }
            
            if (-not $global:WebSocketClient.Connected) {
                Write-Host -ForegroundColor "Red" "PowerShell was unable to connect to TMConsole WebSocketServer at $($WebSocketServer):$($WebSocketPort)"
                throw "PowerShell was unable to connect to TMConsole WebSocketServer at $($WebSocketServer):$($WebSocketPort)"
            }
            
            ## Warm up each runspace so the modules load
            1..$MaxRunspaces | ForEach-Object {
                $PrepShell = [powershell]::Create()
                $PrepShell.RunspacePool = $Global:RunspacePool
                $Null = $PrepShell.AddScript('').Invoke()
            }
            
            ##
            ## Perform primary loop,
            ## Receives ActionRequets and TerminateTask items from Angular
            ## Sends UI update messages to Angular
            ## Manages RunspacePool
            ##
            $LastBatchSend = [datetime]::UtcNow
            $LastRunningActionCount = -1
            $MinBatchSize = 5
            $MaxBatchSize = 50
            $MaxBatchLatency = 50
            $SleepInterval = 10
            $MessageBatch = [System.Collections.Generic.List[object]]@()

            do {
                
                ## Check for any completed runspaces
                foreach ($TMTaskID in $Global:TMConsole.Runspaces.PSObject.Properties.Name) {
                    $TMRunspace = $Global:TMConsole.Runspaces.$TMTaskID
                    if ($TMRunspace.AsyncHandle.IsCompleted) {
                        if ($TMRunspace.Runspace.HadErrors) {
                            ## Capture the error message to send to TMConsole
                            # $InvocationError = $TMRunspace.Runspace.Streams.Error[0].Exception.Message
                            $InvocationError = $TMRunspace.Runspace.Streams.Information[-1].MessageData.Message

                            ## Send the StatusMessages to TMConsole
                            $Global:TMConsole.TaskProgress.Enqueue([PSCustomObject]@{
                                    TMTaskId = $TMTaskId
                                    Type     = 'Progress'
                                    Message  = [PSCustomObject]@{
                                        ActivityId        = 0
                                        ParentActivityId  = -1
                                        Activity          = 'Task Failed'
                                        CurrentOperation  = 'Failed'
                                        StatusDescription = ''
                                        PercentComplete   = 100
                                        SecondsRemaining  = -1
                                        RecordType        = 2
                                    }
                                }
                            )
                            $Global:TMConsole.TaskChanges.Enqueue(
                                [PSCustomObject]@{
                                    TMTaskId     = $TMTaskId
                                    TMTaskNumber = $TMRunspace.TMTaskNumber
                                    Type         = 'Error'
                                    Message      = $InvocationError
                                }
                            )
                            $Global:TMConsole.TaskChanges.Enqueue(
                                [PSCustomObject]@{
                                    TMTaskId     = $TMTaskId
                                    TMTaskNumber = $TMRunspace.TMTaskNumber
                                    Type         = 'TaskFailed'
                                    Message      = $InvocationError
                                }
                            )
                        }
                        else {
                            $Global:TMConsole.TaskProgress.Enqueue( 
                                [PSCustomObject]@{
                                    TMTaskId = $TMTaskID
                                    Type     = 'Progress'
                                    Message  = [PSCustomObject]@{
                                        ActivityId        = 0
                                        ParentActivityId  = -1
                                        Activity          = 'Task Completed'
                                        CurrentOperation  = 'Complete'
                                        StatusDescription = ''
                                        PercentComplete   = 100
                                        SecondsRemaining  = -1
                                        RecordType        = 1
                                    }
                                }
                            )
                            $Global:TMConsole.TaskChanges.Enqueue( 
                                [PSCustomObject]@{
                                    TMTaskId = $TMTaskId
                                    Type     = 'TaskCompleted'
                                }
                            )
                        }
                        $Global:TMConsole.RemoveRunspaces.Enqueue($TMTaskId)
                    }
                }

                ## Collect all of the Task State Change tokens
                while ($Global:TMConsole.RemoveRunspaces.Count) {
                    $TMTaskId = $null
                    if ($Global:TMConsole.RemoveRunspaces.TryDequeue([ref]$TMTaskId)) {

                        try {
                            $TMRunspace = $Global:TMConsole.Runspaces.$TMTaskID
                            [void]($TMRunspace.Runspace.Streams.Information.Clear())
                            [void]($TMRunspace.Runspace.Streams.Error.Clear())
                            [void]($TMRunspace.Runspace.Streams.Warning.Clear())
                            [void]($TMRunspace.Runspace.Streams.Verbose.Clear())
                            [void]($TMRunspace.Runspace.Streams.Debug.Clear())
                            [void]($TMRunspace.Runspace.Streams.Progress.Clear())
                        }
                        catch {
                            ## Errors are present in this output but are not ours to handle here
                        }
                        try {
                            [void]($TMRunspace.Runspace.BeginStop($Null, $TMRunspace.AsyncHandle))
                        }
                        catch {
                            ## Errors are present in this output but are not ours to handle here
                        }
                        try {
                            [void]($TMRunspace.Runspace.Dispose())
                            $TMRunspace.Runspace = $null
                            $TMRunspace.AsyncHandle = $null
                        }
                        catch {
                            ## Errors are present in this output but are not ours to handle here
                        }
                        if ($Global:TMConsole.Runspaces.PSObject.Properties.Name -contains $TMTaskId) {
                            $Global:TMConsole.Runspaces.PSObject.Properties.Remove($TMTaskId)
                        }
                        if ([TMCRunspaceDirectory]::ActiveRunspaces.Keys -contains $TMTaskId) {
                            [void]([TMCRunspaceDirectory]::ActiveRunspaces.TryRemove($TMTaskId, [ref]$null))
                        }
                    }
                }

                ## Check if the runspace count has changed
                if ($LastRunningActionCount -ne $Global:TMConsole.Runspaces.PSObject.Properties.Name.count) {
                    $LastRunningActionCount = $Global:TMConsole.Runspaces.PSObject.Properties.Name.count
                    $ActionCountLabel = switch ($LastRunningActionCount) {
                        0 { '0 Actions Running' }
                        1 { '1 Action Running' }
                        Default { "$([int]$LastRunningActionCount) Actions Running" }
                    }   
                    [void]($MessageBatch.Add([PSCustomObject]@{
                                Type    = 'powershell-server-status'
                                Message = @{
                                    connectionStatus = 'Connected'
                                    serverName       = $WebSocketServer
                                    serverStatus     = $ActionCountLabel
                                }
                            }
                        )
                    )
                }
                
                ##
                ## Priority Messages, always get sent when available
                ##
              
                ## Collect all of the Task State Change tokens
                while ($Global:TMConsole.TaskChanges.Count) {
                    $message = $null
                    if ($Global:TMConsole.TaskChanges.TryDequeue([ref]$message)) {
                        [void]($MessageBatch.Add($message))
                    }
                }
                
                ## Collect all of the System Error tokens
                while ($Global:TMConsole.SystemErrors.Count) {
                    $message = $null
                    if ($Global:TMConsole.SystemErrors.TryDequeue([ref]$message)) {
                        [void]($MessageBatch.Add($message))
                        Write-Warning $Message
                    }
                }

                if ($MessageBatch.Count -gt 0) {
                    [void]($global:WebSocketClient.SendAsync(
                            ($MessageBatch | ConvertTo-Json -EnumsAsStrings -Compress), 
                            [System.Net.WebSockets.WebSocketMessageType]::Text, 
                            $global:CancellationToken
                        )
                    )
                    [void]($MessageBatch.Clear())
                }

                ##
                ## Aggregate Console Messages and load in blocks
                ##
                $Now = [datetime]::UtcNow
                $Elapsed = ($Now - $LastBatchSend).TotalMilliseconds

                ## Collect ConsoleOutput (Fabricated ones from Pipeline, needs updating)
                if ($Global:TMConsole.TaskOutput.count -gt $MinBatchSize -or $Elapsed -gt $MaxBatchLatency) {
                    for ($i = 0; $i -lt $MinBatchSize; $i++) {
                        $message = $null
                        if ($Global:TMConsole.TaskOutput.TryDequeue([ref]$message)) {
                            # Write-Host ($message | ConvertTo-Json -EnumsAsStrings -Compress) -Foreground Yellow
                            [void]($MessageBatch.Add($message))
                        }
                    }
                }
                
                ## Collect Progress Output
                if (
                    ($MessageBatch.Count -gt 0) -or    
                    ($Global:TMConsole.TaskProgress.count -gt $MinBatchSize) -or 
                    ($Elapsed -gt $MaxBatchLatency)
                ) {
                    while ($Global:TMConsole.TaskProgress.Count) {
                        $message = $null
                        if ($Global:TMConsole.TaskProgress.TryDequeue([ref]$message)) {
                            [void]($MessageBatch.Add($message))
                        }
                    }
                }

                if ($MessageBatch.Count -gt 0) {
                    [void]($global:WebSocketClient.SendAsync(
                            ($MessageBatch | ConvertTo-Json -EnumsAsStrings -Compress), 
                            [System.Net.WebSockets.WebSocketMessageType]::Text, 
                            $global:CancellationToken
                        )
                    )
                    [void]($MessageBatch.Clear())
                    $LastBatchSend = $Now
                }

                ## TMC TMCPSHost Collection
                $message = $null
                while ([TMCPSHost]::OutputQueue.TryDequeue([ref]$message)) {
                    [void]($MessageBatch.Add($message))
                    if ($MessageBatch.Count -gt $MaxBatchSize) { break }
                }
                    
                if ($MessageBatch.Count -gt 0) {
                    [void]($global:WebSocketClient.SendAsync(
                            ($MessageBatch | ConvertTo-Json -EnumsAsStrings -Compress), 
                            [System.Net.WebSockets.WebSocketMessageType]::Text, 
                            $global:CancellationToken
                        )
                    )
                    [void]($MessageBatch.Clear())
                    $LastBatchSend = $Now
                }

                Start-Sleep -Milliseconds $SleepInterval

            } until (-Not $global:WebSocketClient.Connected)
        }
        catch {
            Write-Host -ForegroundColor "Red" "FAILURE IN WSS Client"
            throw $_
        }
        finally {
            
            $EventSubscribers = Get-EventSubscriber -SourceIdentifier "TM*" | Foreach-Object
            foreach ($TMEventSubscriber in $EventSubscribers) {
                Unregister-Event -Force -SubscriptionId $TMEventSubscriber.SubscriptionId
                $TMEventSubscriber.Action.Dispose()
                $TMEventSubscriber = $null
            }
            
            ## Disconnect and close down
            if ($global:WebSocketClient) {
                if ($global:WebSocketClient.Connected) {
                    $global:WebSocketClient.Stop()
                }
                $global:WebSocketClient.Dispose()
            }
        }
    }
}