Alerts/ServerRake.Alerts.ps1
|
################## ### ALERTING ### ################## # GOOGLE CHAT function Invoke-GoogleAlertDiskSpace { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [String[]] $ComputerName, [Parameter(Mandatory = $true)] [String[]] $Channels ) $thresholdPercent = 20 $remoteScript = { param($thresholdPercent) $drives = Get-CimInstance Win32_LogicalDisk -Filter "DriveType=3" | Select-Object DeviceID, @{Name = 'SizeGB'; Expression = { [math]::Round($_.Size / 1GB, 1) } }, @{Name = 'FreeGB'; Expression = { [math]::Round($_.FreeSpace / 1GB, 1) } }, @{Name = 'FreePct'; Expression = { [math]::Round(($_.FreeSpace / $_.Size) * 100, 1) } } $lowDrives = $drives | ? { $($_.FreePct) -lt $thresholdPercent -and $($_.DeviceID) -ne 'P:' } if ($lowDrives) { $response = [PSCustomObject]@{ Server = $ENV:COMPUTERNAME Message = $lowDrives | % { "$($_.DeviceID) --- $($_.FreeGB) GB free of $($_.SizeGB) GB ($($_.FreePct)%)" } } return $response } else { Write-Host "All disks on [$ENV:COMPUTERNAME] have more than [$thresholdPercent%] free space." return } } $response = Send-RemoteScript -ComputerName $ComputerName -RemoteScript $remoteScript -ArgumentList $thresholdPercent if ($response) { $Parameters = @{ Response = $response Title = "⚠️ LOW DISK SPACE ALERT" Severity = "Medium to High" Summary = "The following servers have drives below the [$thresholdPercent%] free space threshold" Action = "Assess the following drives and allocate more space as needed:" Channels = $Channels } <<<<<<< HEAD Send-GoogleMessage @Parameters ======= Send-GoogleAlert @Parameters >>>>>>> uat return } else { return } } function Invoke-GoogleAlertCPUUtilization { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [String[]] $ComputerName, [Parameter(Mandatory = $true)] [String[]] $Channels ) $sampleInterval = 5 $thresholdPercent = 10 $remoteScript = { param ($sampleInterval, $thresholdPercent) $counters = @( '\Processor(_Total)\% Processor Time', '\Process(*)\% Processor Time' ) $samples = (Get-Counter -Counter $Counters -SampleInterval $sampleInterval).CounterSamples | Select-Object -First 11 $results = $samples | % { [PSCustomObject] @{'Process' = $_.InstanceName; 'CPU%' = ($_.CookedValue / 100).ToString('P1') } } $processor = $results[0].'CPU%' if ($results[0].CookedValue -gt $thresholdPercent) { $response = [PSCustomObject]@{ Server = $ENV:COMPUTERNAME Message = $results[1..($results.Count - 1)] | % { "$($_.'CPU%') ::: Process Name ::: $($_.Process)" } } return $response } else { Write-Host "CPU on [$ENV:COMPUTERNAME] running below [$thresholdPercent%] threshold [Currently, $($processor)]." return } } $response = Send-RemoteScript -ComputerName $ComputerName -RemoteScript $remoteScript -ArgumentList $sampleInterval, $thresholdPercent if ($response) { $Parameters = @{ Response = $response Title = "🚩 HIGH CPU USAGE ALERT" Severity = "High" Summary = "The following server CPUs are operating at or above a [$thresholdPercent%] threshold" Action = "Check the following servers and listed processes for abnormal utilization:" Channels = $Channels } <<<<<<< HEAD Send-GoogleMessage @Parameters ======= Send-GoogleAlert @Parameters >>>>>>> uat return } else { return } } function Invoke-GoogleAlertRubrikService { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [String[]] $ComputerName, [Parameter(Mandatory = $true)] [String[]] $Channels ) $remoteScript = { $services = Get-CimInstance -ClassName Win32_Service -Filter "Name = 'Rubrik Backup Service'" | Select-Object Name, StartName, State $messages = $null if ($services) { foreach ($s in $services) { if ($s.State -eq "Stopped" -or $s.StartName -notlike "*svc_rubrik*") { Write-Host "[$ENV:COMPUTERNAME] - [$($s.Name), as '$($s.StartName)'] - State: [$($s.State)]." -ForegroundColor Magenta $message = "$($s.State), running as '$($s.StartName)'" $messages += $message.Replace('\', '/') } else { Write-Host "[$ENV:COMPUTERNAME] - [$($s.Name), as '$($s.StartName)'] - State: [$($s.State)]." } } } else { Write-Host "Rubrik Backup Service was not detected on [$ENV:COMPUTERNAME]." -ForegroundColor Magenta $messages += "Rubrik Backup Service not detected." } if ($messages) { $response = [PSCustomObject]@{ Server = $ENV:COMPUTERNAME Message = $messages } return $response } else { return } } $response = Send-RemoteScript -ComputerName $ComputerName -RemoteScript $remoteScript $response = $response | sort -Property Message -Descending if ($response) { $Parameters = @{ Response = $response Title = "🟡 SERVICE: RUBRIK BACKUP SERVICE" Severity = "High" Summary = "This service requires attention." Action = "Please review the details listed for the following servers:" Channels = $Channels } <<<<<<< HEAD Send-GoogleMessage @Parameters ======= Send-GoogleAlert @Parameters >>>>>>> uat return } else { return } } function Invoke-GoogleAlertNetworkingChange { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [String[]] $ComputerName, [Parameter(Mandatory = $true)] [String[]] $Channels ) $remoteScript = { $events = Get-WinEvent -FilterHashtable @{ LogName = 'Microsoft-Windows-NetworkProfile/Operational' StartTime = (Get-Date).AddDays(-1) } -ErrorAction SilentlyContinue if ($events) { $response = [PSCustomObject]@{ Server = $ENV:COMPUTERNAME Message = $events | % { "$($_.TimeCreated) - $($_.Id)`n$($_.Message)`n" } } return $response } else { Write-Host "No networking changes were detected on [$ENV:COMPUTERNAME] in the past 24 hours." return } } $response = Send-RemoteScript -ComputerName $ComputerName -RemoteScript $remoteScript -ArgumentList $HAL_9000, $TEST_URI if ($response) { $Parameters = @{ Response = $response Title = "🟡 NETWORK CHANGES DETECTED" Severity = "Medium to High" Summary = "Networking changes have been detected within the past 24 hours." Action = "Investigate the following networking log messages:" Channels = $Channels } Send-GoogleAlert @Parameters return } else { return } } function Send-GoogleAlert { [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [Object[]] $Response, [Parameter(Mandatory = $true)] [String] $Title, [Parameter(Mandatory = $true)] [String] $Severity, [Parameter(Mandatory = $true)] [String] $Summary, [Parameter(Mandatory = $true)] [String] $Action, [Parameter(Mandatory = $true)] [String[]] $Channels ) $message = $null foreach ($server in $Response) { if ($server.Server) { $message += "$($server.Server)`n" $message += $server.Message | % { "$_`n" } $message += "`n" } } $jsonPayload = @" { "cardsV2": [ { "cardId": "000", "card": { "header": { "title": "$Title", "subtitle": ">>>> Severity: $Severity" }, "sections": [ { "header": "Event Details", "collapsible": false, "uncollapsibleWidgetsCount": 1, "widgets": [ { "decoratedText": { "icon": { "materialIcon": { "name": "error" } }, "topLabel": "$Summary", "bottomLabel": "$Action" } }, { "textParagraph": { "text": "$message", "maxLines": 3 } } ] } ] } } ] } "@ if ($message) { try { $Channels | % { Invoke-RestMethod -Uri $_ -Method Post -Body $jsonPayload -ContentType "application/json; charset=utf-8" | Out-Null } $Response | % { if ($_.Server) { Write-Host "Alert [$title] sent for [$($_.Server)]" -ForegroundColor Magenta } } return } catch { $_.Exception } } else { return } } |