cmclustermaintenance.psm1
|
$script:localizedData = Import-LocalizedData -BaseDirectory "$PSScriptRoot\Docs\en-US" -FileName 'CmClusterMaintenance.strings.psd1' function Get-FormattedDate { [CmdletBinding()] param( [Parameter()] [String] $Format = (Get-Culture).DateTimeFormat.UniversalSortableDateTimePattern ) # Returning current time based on UniversalSortableDateTimePattern if (-not $PSBoundParameters.ContainsKey('Format')) { $PSBoundParameters.Add('Format', $Format) } Get-Date @PSBoundParameters } function Resume-CmClusterNode { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [String] $Cluster, [Parameter(Mandatory = $true)] [String] $Name ) Write-Verbose -Message ($script:localizedData.startingOnTarget -f $(Get-FormattedDate), 'Resume-CmClusterNode', $Cluster.ToUpper(), $Name.ToUpper()) try { # Try to suspend the target node $clusterNodeParams = $PSBoundParameters $clusterNodeParams.ErrorAction = 'Stop' $clusterNodeParams.Verbose = $false Resume-ClusterNode @clusterNodeParams } catch { Write-Verbose -Message ($script:localizedData.failedToResume -f $(Get-FormattedDate), $Cluster.ToUpper(), $Name.ToUpper()) throw $_ } } function Start-CmClusterNode { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [String] $Cluster, [Parameter(Mandatory = $true)] [String] $Name ) Write-Verbose -Message ($script:localizedData.startingOnTarget -f $(Get-FormattedDate), 'Start-CmClusterNode', $Cluster.ToUpper(), $Name.ToUpper()) try { # Try to suspend the target node $clusterNodeParams = $PSBoundParameters $clusterNodeParams.ErrorAction = 'Stop' $clusterNodeParams.Verbose = $false Start-ClusterNode @clusterNodeParams } catch { Write-Verbose -Message ($script:localizedData.failedToStart -f $(Get-FormattedDate), $Cluster.ToUpper(), $Name.ToUpper()) throw $_ } } function Stop-CmClusterNode { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [String] $Cluster, [Parameter(Mandatory = $true)] [String] $Name ) Write-Verbose -Message ($script:localizedData.startingOnTarget -f $(Get-FormattedDate), 'Stop-CmClusterNode', $Cluster.ToUpper(), $Name.ToUpper()) try { # Try to suspend the target node $clusterNodeParams = $PSBoundParameters $clusterNodeParams.ErrorAction = 'Stop' $clusterNodeParams.Verbose = $false Stop-ClusterNode @clusterNodeParams } catch { Write-Verbose -Message ($script:localizedData.failedToStop-f $(Get-FormattedDate), $Cluster.ToUpper(), $Name.ToUpper()) throw $_ } } function Suspend-CmClusterNode { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [String] $Cluster, [Parameter(Mandatory = $true)] [String] $Name, [Parameter(Mandatory = $false)] [Switch] $ForceDrain ) Write-Verbose -Message ($script:localizedData.startingOnTarget -f $(Get-FormattedDate), 'Suspend-CmClusterNode', $Cluster.ToUpper(), $Name.ToUpper()) try { # Try to suspend the target node $clusterNodeParams = $PSBoundParameters $clusterNodeParams.Drain = $true $clusterNodeParams.ErrorAction = 'Stop' $clusterNodeParams.Verbose = $false Suspend-ClusterNode @clusterNodeParams } catch { Write-Verbose -Message ($script:localizedData.failedToSuspend-f $(Get-FormattedDate), $Cluster.ToUpper(), $Name.ToUpper()) throw $_ } } function Test-CmClusterNode { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [String] $Cluster, [Parameter(Mandatory = $true)] [String] $Name, [Parameter(Mandatory = $true)] [ValidateSet('Resume', 'Start', 'Stop', 'Suspend')] [String] $TestType, [Parameter(Mandatory = $false)] [Int32] $TimeOut = 600 ) Write-Verbose -Message ($script:localizedData.startingTestType -f $(Get-FormattedDate), $Cluster.ToUpper(), $Name.ToUpper(), $TestType) # Removing timeout & TestType / Adding ErrorAction for Get-ClusterNode $PSBoundParameters.Remove('TestType') | Out-Null $PSBoundParameters.Remove('TimeOut') | Out-Null $PSBoundParameters.ErrorAction = 'Stop' # Setting stopwatch to capture elapsed time so that we can break out of the while based on the TimeOut value $stopWatch = [System.Diagnostics.Stopwatch]::StartNew() # Setting testResults to false $testResults = $false # Looping until $TimeOut expires or testResults equals $true; :breakOut is a label for the while statement, used with the break statement :breakOut while ($testResults -ne $true) { if ($stopWatch.Elapsed.TotalSeconds -ge $TimeOut) { $timerExpired = $true break breakOut } # try to query the node drain status from the clustered node try { $nodeStatus = Get-ClusterNode @PSBoundParameters } catch { Write-Verbose -Message ($script:localizedData.failedToQuery -f $(Get-FormattedDate), $Cluster.ToUpper(), $Name.ToUpper()) throw $_ } # Switching on TestType and returning $testResults switch ($TestType) { 'Resume' { if ($nodeStatus.State -eq 'Up') { # setting testResults to true and breaking out of the while $testResults = $true break breakOut } } 'Start' { if ($nodeStatus.State -eq 'Paused') { # setting testResults to true and breaking out of the while $testResults = $true break breakOut } } 'Stop' { # if the node State is 'Down' setting $testResults and breaking out of the while if ($nodeStatus.State -eq 'Down') { # setting testResults to true and breaking out of the while $testResults = $true break breakOut } } 'Suspend' { # if the node State is 'Paused' using switch to determine DrainStatus and setting testResults bool if ($nodeStatus.State -eq 'Paused') { # CLUSTER_NODE_DRAIN_STATUS (MSDN) - https://msdn.microsoft.com/en-us/library/dn622912(v=vs.85).aspx switch ($nodeStatus.DrainStatus) { 'NotInitiated' { # setting testResults to false and breaking out of the switch $testResults = $false break } 'InProgress' { # setting testResults to false and breaking out of the switch $testResults = $false break } 'Completed' { # setting testResults to true and breaking out of the while $testResults = $true break breakOut } 'Failed' { # setting testResults to false and breaking out of the while $testResults = $false break breakOut } Default { # if the above four conditions do not apply; setting testResults to false $testResults = $false } } } } } # sleeping to limit the queries for node status Start-Sleep -Milliseconds 750 } # return true/false to the calling function Write-Verbose -Message ($script:localizedData.currentNodeState -f $(Get-FormattedDate), $Cluster.ToUpper(), $Name.ToUpper(), $($nodeStatus.State), $($nodeStatus.DrainStatus)) if ($timerExpired) { Write-Verbose -Message ($script:localizedData.timerExpired -f $(Get-FormattedDate), $TimeOut, $Cluster.ToUpper(), $Name.ToUpper()) } return $testResults } <# .SYNOPSIS Invoke-CmClusterMaintenance is used to start node maintenance on all nodes in a cluster, coordinating node drain and reboot (optional). .DESCRIPTION Invoke-CmClusterMaintenance is used to start node maintenance on all nodes in a cluster, coordinating node drain and reboot (optional). The function will query all cluster nodes using Get-ClusterNode with the given cluster name. Once it receives the cluster nodes it will loop through all nodes within the cluster, pausing, stopping, performing an action (specified via the Scriptblock parameter), rebooting (if specified via the Reboot switch parameter), starting, resuming and detecting success in each step of the coordinated process. If any one of the steps is not successful, the entire coordinated process will fail, if the Verbose parameter is used, more detial around possible causes for the failure can be used for troubleshooting purposes. .PARAMETER Cluster Specifies the name of the cluster on which to run this function. The function will loop through all nodes of the specified cluster invoking the action contained within the scriptblock parameter. .PARAMETER ForceDrain Specifies that workloads are moved from a node even in the case of an error. .PARAMETER ScriptBlock Specifies the commands to run. Enclose the commands in braces ( { } ) to create a script block. This parameter is required. By default, any variables in the command are evaluated on the remote computer. .PARAMETER ArgumentList Supplies the values of local variables in the command. The variables in the command are replaced by these values before the command is run on the remote computer. Enter the values in a comma-separated list. Values are associated with variables in the order that they are listed. The alias for ArgumentList is "Args". The values in ArgumentList can be actual values, such as "1024", or they can be references to local variables, such as "$max". To use local variables in a command, use the following command format: {param($<name1>[, $<name2>]...) <command-with-local-variables>} -ArgumentList <value> -or- <local-variable> or $localVariable = 'Data is stored here' Invoke-CmClusterMaintenance -Cluster HV-CLUS1 -ScriptBlock {$using:localVariable} -ArgumentList $localVariable -RebootNode The "param" keyword lists the local variables that are used in the command. The ArgumentList parameter supplies the values of the variables, in the order that they are listed. .PARAMETER TimeOut Specifies the number of seconds in which the function will wait for the drain action to complete for a node. If the TimeOut expires, then the function will fail for the cluster. *** In it's current state, if a drain fails, then the function will stop and take no further action for the entire cluster if you wish to modify this behavior, it is possible, but will require additional logic/modification. *** .PARAMETER RebootNode Specifying the RebootNode switch parameter will cause the node to reboot after the scriptblock contents is executed against the node. .EXAMPLE $scriptBlock = {"Action to perform on $env:ComputerName"} Invoke-CmClusterMaintenance -Cluster HV-CLUS1 -ScriptBlock $scriptBlock -RebootNode This example will execute the contents of the scriptblock then reboot the node, one at a time, for all nodes in the HV-CLUS1 cluster. .NOTES Created by Brian Wilhite #> function Invoke-CmClusterMaintenance { [CmdletBinding(DefaultParameterSetName = 'ScriptBlock')] param( [Parameter(Mandatory = $true)] [String] $Cluster, [Parameter()] [Switch] $ForceDrain, [Parameter(Mandatory = $true, ParameterSetName = 'ScriptBlock')] [ScriptBlock] $ScriptBlock, [Parameter(Mandatory = $true, ParameterSetName = 'FilePath')] [ValidateScript({Test-Path -Path $_})] [String] $FilePath, [Parameter()] [System.Object[]] $ArgumentList, [Parameter()] [Int32] $TimeOut = 600, [Parameter()] [Switch] $RebootNode ) Write-Verbose -Message ($script:localizedData.startingOnCluster -f $(Get-FormattedDate), 'Invoke-CmClusterMaintenance', $Cluster.ToUpper()) # Get Cluster Nodes try { $clusterNodes = Get-ClusterNode -Cluster $Cluster -ErrorAction Stop } catch { throw $_ } # Removing RebootNode, Scriptblock, FilePath & TimeOut from PSBoundParams $PSBoundParameters.Remove('RebootNode') | Out-Null $PSBoundParameters.Remove('ScriptBlock') | Out-Null $PSBoundParameters.Remove('FilePath') | Out-Null $PSBoundParameters.Remove('ArgumentList') | Out-Null $PSBoundParameters.Remove('TimeOut') | Out-Null # If the ForceDrain parameter was specified remove it from PSBoundParams and add ForceDrain = $true to the hashtable for splatting if ($PSBoundParameters.Remove('ForceDrain')) { [hashtable]$suspendCmClusterNodeParams = $PSBoundParameters $suspendCmClusterNodeParams.ForceDrain = $true } else { $suspendCmClusterNodeParams = $PSBoundParameters } # Loop, using for to track cluster progress for ($i = 0; $i -lt $clusterNodes.Count; $i++) { # Setting up the Write-Progress parameter - Splatting $writeProgressPrcent = [Math]::Round($i / $clusterNodes.Count * 100) $writeProgressParams = @{ Activity = $script:localizedData.progActvFirstMsg -f $Cluster.ToUpper(), $clusterNodes[$i].Name Status = $script:localizedData.progStatusMsg -f $($i + 1), $clusterNodes.Count, $writeProgressPrcent PercentComplete = $writeProgressPrcent Id = 1 } Write-Progress @writeProgressParams # Adding the current cluster node to PSBoundParameters/SuspendCmClusterNodeParams $PSBoundParameters.Name = $clusterNodes[$i].Name $suspendCmClusterNodeParams.Name = $clusterNodes[$i].Name try { # Suspending current cluster node Write-Progress -Activity ($script:localizedData.progActvSecondMsg -f $clusterNodes[$i].Name) -ParentId 1 -PercentComplete (1 / 8 * 100) Suspend-CmClusterNode @suspendCmClusterNodeParams | Out-Null # Testing successful cluster node SUSPEND operation Write-Progress -Activity ($script:localizedData.progActvThirdMsg -f $clusterNodes[$i].Name) -ParentId 1 -PercentComplete (2 / 8 * 100) if (Test-CmClusterNode @PSBoundParameters -TestType Suspend -TimeOut $TimeOut) { # If cluster node suspend was successful, stop the cluster node Write-Progress -Activity ($script:localizedData.progActvFourthMsg -f $clusterNodes[$i].Name) -ParentId 1 -PercentComplete (3 / 8 * 100) Stop-CmClusterNode @PSBoundParameters | Out-Null } else { # Otherwise throw a time stamped error throw ($script:localizedData.failedfunction -f $(Get-FormattedDate), 'Suspend-CmClusterNode', $Cluster.ToUpper(), $PSBoundParameters.Name) } # Testing successful cluster node STOP operation Write-Progress -Activity ($script:localizedData.progActvFifthMsg -f $clusterNodes[$i].Name) -ParentId 1 -PercentComplete (4 / 8 * 100) if (Test-CmClusterNode @PSBoundParameters -TestType Stop -TimeOut $TimeOut) { # Setting up Invoke-Command parameters Write-Progress -Activity ($script:localizedData.progActvSixthMsg -f $clusterNodes[$i].Name) -ParentId 1 -PercentComplete (5 / 8 * 100) $invokeCmdParams = @{ ComputerName = $clusterNodes[$i].Name ErrorAction = 'Stop' } # If the ScriptBlock param was specified, then add it to the invokeCmdParams hashtable for splatting if ($ScriptBlock) { $invokeCmdParams.ScriptBlock = $ScriptBlock } # If the FilePath param was specified, then add it to the invokeCmdParams hashtable for splatting if ($FilePath) { $invokeCmdParams.FilePath = $FilePath } # If the ArgumentList param was specified, then add it to the invokeCmdParams hashtable for splatting if ($ArgumentList) { $invokeCmdParams.ArgumentList = $ArgumentList } # Invoking the specified scriptblock and parameters on the current cluster node Invoke-Command @invokeCmdParams # If the RebootNode param was specified, then Restart-Computer on the current cluster node and wait for PowerShell remoting on the target node before continuing if ($RebootNode) { Write-Verbose -Message ($script:localizedData.startingOnTarget -f $(Get-FormattedDate), 'Restart-Computer', $Cluster.ToUpper(), $Name) Restart-Computer -ComputerName $PSBoundParameters.Name -Wait -For PowerShell -Force } } else { throw ($script:localizedData.failedfunction -f $(Get-FormattedDate), 'Stop-CmClusterNode', $Cluster.ToUpper(), $PSBoundParameters.Name) } # Start the cluster service on the current cluster node Write-Progress -Activity ($script:localizedData.progActvSeventhMsg -f $clusterNodes[$i].Name) -ParentId 1 -PercentComplete (6 / 8 * 100) Start-CmClusterNode @PSBoundParameters | Out-Null # Testing successful cluster service START operation if (Test-CmClusterNode @PSBoundParameters -TestType Start -TimeOut $TimeOut) { # if cluster node START operation was successful, then attempt to resume the cluster node (Unpause) Write-Progress -Activity ($script:localizedData.progActvEightMsg -f $clusterNodes[$i].Name) -ParentId 1 -PercentComplete (7 / 8 * 100) Resume-CmClusterNode @PSBoundParameters | Out-Null } else { throw ($script:localizedData.failedfunction -f $(Get-FormattedDate), 'Start-CmClusterNode', $Cluster.ToUpper(), $PSBoundParameters.Name) } # Testing successful cluster RESUME operation Write-Progress -Activity ($script:localizedData.progActvNinthMsg -f $clusterNodes[$i].Name) -ParentId 1 -PercentComplete (8 / 8 * 100) if (Test-CmClusterNode @PSBoundParameters -TestType Resume -TimeOut $TimeOut) { Write-Verbose -Message ($script:localizedData.invokeCmComplete -f $(Get-FormattedDate), $Cluster.ToUpper(), $PSBoundParameters.Name) } else { throw ($script:localizedData.failedfunction -f $(Get-FormattedDate), 'Resume-CmClusterNode', $Cluster.ToUpper(), $PSBoundParameters.Name) } } catch { Write-Verbose -Message ($script:localizedData.invokeCmFailed -f $(Get-FormattedDate), $Cluster.ToUpper(), $PSBoundParameters.Name) throw $_ } } } |