RobustCloudCommand.psm1
|
Function Start-RobustCloudCommand { <# .SYNOPSIS Generic wrapper script that tries to ensure that a script block successfully finishes execution in O365 against a large object count. Works well with intense operations that may cause throttling .DESCRIPTION Wrapper script that tries to ensure that a script block successfully finishes execution in O365 against a large object count. It accomplishs this by doing the following: * Monitors the health of the Remote powershell session and restarts it as needed. * Restarts the session every X number seconds to ensure a valid connection. * Attempts to work past session related errors and will skip objects that it can't process. * Attempts to calculate throttle exhaustion and sleep a sufficient time to allow throttle recovery .PARAMETER AutomaticThrottle Calculated value based on your tenants powershell recharge rate. You tenant recharge rate can be calculated using a Micro Delay Warning message. Look for the following line in your Micro Delay Warning Message Balance: -1608289/2160000/-3000000 The middle value is the recharge rate. Divide this value by the number of milliseconds in an hour (3600000) And subtract the result from 1 to get your AutomaticThrottle value 1 - (2160000 / 3600000) = 0.4 Default Value is .25 .PARAMETER IdentifyingProperty What property of the objects we are processing that will be used to identify them in the log file and host If the value is not set by the user the script will attempt to determine if one of the following properties is present "DisplayName","Name","Identity","PrimarySMTPAddress","Alias","GUID" If the value is not set and we are not able to match a well known property the script will generate an error and terminate. .PARAMETER LogFile Location and file name for the log file. .PARAMETER ManualThrottle Manual delay of X number of milliseconds to sleep between each cmdlets call. Should only be used if the AutomaticThrottle isn't working to introduce sufficent delay to prevent Micro Delays .PARAMETER NonInteractive Suppresses output to the screen. All output will still be in the log file. .PARAMETER Recipients Array of objects to operate on. This can be mailboxes or any other set of objects. Input must be an array! Anything comming in from the array can be accessed in the script block using $input.property .PARAMETER ResetSeconds How many seconds to run the script block before we rebuild the session with O365. .PARAMETER ScriptBlock The script that you want to robustly execute against the array of objects. The Recipient objects will be provided to the cmdlets in the script block and can be accessed with $input as if you were pipelining the object. .LINK https://github.com/Canthv0/RobustCloudCommand .OUTPUTS Creates the log file specified in -logfile. Logfile contains a record of all actions taken by the script. .EXAMPLE invoke-command -scriptblock {Get-mailbox -resultsize unlimited | select-object -property Displayname,PrimarySMTPAddress,Identity} -session (get-pssession) | export-csv c:\temp\mbx.csv $mbx = import-csv c:\temp\mbx.csv $cred = get-Credential .\Start-RobustCloudCommand.ps1 -Credential $cred -recipients $mbx -logfile C:\temp\out.log -ScriptBlock {Set-Clutter -identity $input.PrimarySMTPAddress.tostring() -enable:$false} Gets all mailboxes from the service returning only Displayname,Identity, and PrimarySMTPAddress. Exports the results to a CSV Imports the CSV into a variable Gets your O365 Credential Executes the script setting clutter to off using Legacy Credentials .EXAMPLE invoke-command -scriptblock {Get-mailbox -resultsize unlimited | select-object -property Displayname,PrimarySMTPAddress,Identity} -session (get-pssession) | export-csv c:\temp\recipients.csv $recipients = import-csv c:\temp\recipients.csv Start-RobustCloudCommand -recipients $recipients -logfile C:\temp\out.log -ScriptBlock {Get-MobileDeviceStatistics -mailbox $input.PrimarySMTPAddress.tostring() | Select-Object -Property @{Name = "PrimarySMTPAddress";Expression={$input.PrimarySMTPAddress.tostring()}},DeviceType,LastSuccessSync,FirstSyncTime | Export-Csv c:\temp\stats.csv -Append } Gets All Recipients and exports them to a CSV (for restart ability) Imports the CSV into a variable Executes the script to gather EAS Device statistics and output them to a csv file using ADAL with support for MFA #> Param( [Parameter(Mandatory = $true)] [string]$LogFile, [Parameter(Mandatory = $true)] $Recipients, [Parameter(Mandatory = $true)] [ScriptBlock]$ScriptBlock, $Credential, [int]$ManualThrottle = 0, [double]$ActiveThrottle = .25, [int]$ResetSeconds = 870, [string]$IdentifyingProperty, [Switch]$NonInteractive ) # Turns on strict mode https://technet.microsoft.com/library/03373bbe-2236-42c3-bf17-301632e0c428(v=wps.630).aspx Set-StrictMode -Version 2 $InformationPreference = "Continue" $Global:ErrorActionPreference = "Stop" Write-Log ("Error Action Preference: " + $Global:ErrorActionPreference) # Log the script block for debugging purposes Write-log $ScriptBlock # Setup our first session to O365 $ErrorCount = 0 New-CleanO365Session # Get when we started the script for estimating time to completion $ScriptStartTime = Get-Date [int]$ObjectsProcessed = 0 [int]$ObjectCount = $Recipients.count # If we don't have an identifying property then try to find one if ([string]::IsNullOrEmpty($IdentifyingProperty)) { # Call our function for finding an identifying property and pass in the first recipient object $IdentifyingProperty = Get-ObjectIdentificationProperty -object $Recipients[0] } # Go thru each recipient object and execute the script block foreach ($object in $Recipients) { # Set our initial while statement values $TryCommand = $true $errorcount = 0 $Global:Error.clear() # Try the command 3 times and exit out if we can't get it to work # Record the error and restart the session each time it errors out while ($TryCommand) { Write-log ("Running scriptblock for " + ($object.$IdentifyingProperty).tostring()) # Test our connection and rebuild if needed Test-O365Session # Invoke the script block try { Invoke-Command -InputObject $object -ScriptBlock $ScriptBlock -ErrorAction Stop # Since we didn't get an error don't run again $TryCommand = $false # Increment the object processed count / Estimate time to completion $ObjectsProcessed = Get-EstimatedTimeToCompletion -ProcessedCount $ObjectsProcessed -TotalObjects $ObjectCount -StartTime $ScriptStartTime } catch { # Handle if we keep failing on the object if ($errorcount -ge 3) { Write-Log ("[ERROR] - Object `"" + ($object.$IdentifyingProperty).tostring() + "`" has failed three times!") Write-Log ("[ERROR] - Skipping Object") # Increment the object processed count / Estimate time to completion $ObjectsProcessed = Get-EstimatedTimeToCompletion -ProcessedCount $ObjectsProcessed -StartTime $ScriptStartTime # Set trycommand to false so we abort the while loop $TryCommand = $false } # Otherwise try the command again else { if ($null -eq $Global:Error){ Write-Log "Global Error Null" Write-Log ("Local Error: " + $Error) } else { Write-Log $Global:Error } Write-Log ("Rebuilding session and trying again") $ErrorCount++ # Create a new session in case the error was due to a session issue New-CleanO365Session } } } } Write-Log "Script Complete Destroying PS Sessions" # Destroy any outstanding PS Session Get-PSSession | Remove-PSSession -Confirm:$false $Global:ErrorActionPreference = "Continue" Write-Log ("Error Action Preference: " + $Global:ErrorActionPreference) } # Writes output to a log file with a time date stamp Function Write-Log { Param ([string]$string) # Get the current date [string]$date = Get-Date -Format G # Write everything to our log file ( "[" + $date + "] - " + $string) | Out-File -FilePath $LogFile -Append # If NonInteractive true then suppress host output if (!($NonInteractive)) { Write-Information ( "[" + $date + "] - " + $string) } } # Sleeps X seconds and displays a progress bar Function Start-SleepWithProgress { Param([int]$sleeptime) # Loop Number of seconds you want to sleep For ($i = 0; $i -le $sleeptime; $i++) { $timeleft = ($sleeptime - $i); # Progress bar showing progress of the sleep Write-Progress -Activity "Sleeping" -CurrentOperation "$Timeleft More Seconds" -PercentComplete (($i / $sleeptime) * 100) -Status " " # Sleep 1 second start-sleep 1 } Write-Progress -Completed -Activity "Sleeping" -Status " " } # Setup a new O365 Powershell Session Function New-CleanO365Session { # Destroy any outstanding PS Session Write-Log "Removing all PS Sessions" Get-PSSession | Remove-PSSession -Confirm:$false # Force Garbage collection just to try and keep things more agressively cleaned up due to some issue with large memory footprints [System.GC]::Collect() # Sleep 15s to allow the sessions to tear down fully Write-Log ("Sleeping 15 seconds for Session Tear Down") Start-SleepWithProgress -SleepTime 15 # Clear out all errors $Error.Clear() # Create the session Write-Log "Connecting to Exchange Online" if ($null -eq $Credential){ Connect-EXO } else { Connect-EXOLegacy -Credential $Credential } # Check for an error while creating the session if ($Error.Count -gt 0) { Write-Log "[ERROR] - Error while setting up session" Write-log $Error # Increment our error count so we abort after so many attempts to set up the session $ErrorCount++ # if we have failed to setup the session > 3 times then we need to abort because we are in a failure state if ($ErrorCount -gt 3) { Write-log "[ERROR] - Failed to setup session after multiple tries" Write-log "[ERROR] - Aborting Script" exit } # If we are not aborting then sleep 60s in the hope that the issue is transient Write-Log "Sleeping 60s so that issue can potentially be resolved" Start-SleepWithProgress -sleeptime 60 # Attempt to set up the sesion again New-CleanO365Session } # If the session setup worked then we need to set $errorcount to 0 else { $ErrorCount = 0 } # Set the Start time for the current session Set-Variable -Scope script -Name SessionStartTime -Value (Get-Date) } # Verifies that the connection is healthy # Goes ahead and resets it every $ResetSeconds number of seconds either way Function Test-O365Session { # Get the time that we are working on this object to use later in testing $ObjectTime = Get-Date # Reset and regather our session information $SessionInfo = $null $SessionInfo = Get-PSSession # Make sure we found a session if ($null -eq $SessionInfo) { Write-Log "[ERROR] - No Session Found" Write-log "Recreating Session" New-CleanO365Session } # Make sure it is in an opened state if not log and recreate elseif ($SessionInfo.State -ne "Opened") { Write-Log "[ERROR] - Session not in Open State" Write-log ($SessionInfo | Format-List | Out-String ) Write-log "Recreating Session" New-CleanO365Session } # If we have looped thru objects for an amount of time gt our reset seconds then tear the session down and recreate it elseif (($ObjectTime - $SessionStartTime).totalseconds -gt $ResetSeconds) { Write-Log ("Session Has been active for greater than " + $ResetSeconds + " seconds" ) Write-Log "Rebuilding Connection" # Estimate the throttle delay needed since the last session rebuild # Amount of time the session was allowed to run * our activethrottle value # Divide by 2 to account for network time, script delays, and a fudge factor # Subtract 15s from the results for the amount of time that we spend setting up the session anyway [int]$DelayinSeconds = ((($ResetSeconds * $ActiveThrottle) / 2) - 15) # If the delay is >15s then sleep that amount for throttle to recover if ($DelayinSeconds -gt 0) { Write-Log ("Sleeping " + $DelayinSeconds + " addtional seconds to allow throttle recovery") Start-SleepWithProgress -SleepTime $DelayinSeconds } # If the delay is <15s then the sleep already built into New-CleanO365Session should take care of it else { Write-Log ("Active Delay calculated to be " + ($DelayinSeconds + 15) + " seconds no addtional delay needed") } # new O365 session and reset our object processed count New-CleanO365Session } else { # If session is active and it hasn't been open too long then do nothing and keep going } # If we have a manual throttle value then sleep for that many milliseconds if ($ManualThrottle -gt 0) { Write-log ("Sleeping " + $ManualThrottle + " milliseconds") Start-Sleep -Milliseconds $ManualThrottle } } # If the $identifyingProperty has not been set then we attempt to locate a value for tracking modified objects Function Get-ObjectIdentificationProperty { Param($object) Write-Log "Trying to identify a property for displaying per object progress" # Common properties to check [array]$PropertiesToCheck = "DisplayName", "Name", "Identity", "PrimarySMTPAddress", "Alias", "GUID" # Set our counter to 0 $i = 0 [string]$PropertiesString = $null [bool]$Found = $false # While we haven't found an ID property continue checking while ($found -eq $false) { # If we have gone thru the list then we need to throw an error because we don't have Identity information # Set the string to bogus just to ensure we will exit the while loop if ($i -gt ($PropertiesToCheck.length - 1)) { Write-Log "[ERROR] - Unable to find a common identity parameter in the input object" # Create an error message that has all of the valid property names that we are looking for ForEach ($value in $PropertiesToCheck) { [string]$PropertiesString = $PropertiesString + "`"" + $value + "`", " } $PropertiesString = $PropertiesString.TrimEnd(", ") [string]$errorstring = "Objects does not contain a common identity parameter " + $PropertiesString + " please use -IdentifyingProperty to set the identity value" # Throw error Write-Error -Message $errorstring -ErrorAction Stop } # Get the property we are testing out of our array [string]$Property = $PropertiesToCheck[$i] # Check the properties of the object to see if we have one that matches a well known name # If we have found one set the value to that property if ($null -ne $object.$Property) { Write-log ("Found " + $Property + " to use for displaying per object progress") $found = $true Return $Property } # Increment our position counter $i++ } } # Gather and print out information about how fast the script is running Function Get-EstimatedTimeToCompletion { param([int]$ProcessedCount,[int]$TotalObjects, [datetime]$StartTime) # Increment our count of how many objects we have processed $ProcessedCount++ # Every 100 we need to estimate our completion time and write that out if (($ProcessedCount % 100) -eq 0) { # Get the current date $CurrentDate = Get-Date # Average time per object in seconds $AveragePerObject = (((($CurrentDate) - $StartTime).totalseconds) / $ProcessedCount) # Write out session stats and estimated time to completion Write-Log ("[STATS] - Total Number of Objects: " + $TotalObjects) Write-Log ("[STATS] - Number of Objects processed: " + $ProcessedCount) Write-Log ("[STATS] - Average seconds per object: " + $AveragePerObject) Write-Log ("[STATS] - Estimated completion time: " + $CurrentDate.addseconds((($TotalObjects - $ProcessedCount) * $AveragePerObject))) } # Return number of objects processed so that the variable in incremented return $ProcessedCount } |