Private/Checks/VCenter/Test-VcfVcenterOverallHealth.ps1

# Copyright (c) 2026 Broadcom. All Rights Reserved.
# Broadcom Confidential. The term "Broadcom" refers to Broadcom Inc.
# and/or its subsidiaries.
#
# =============================================================================
#
# SOFTWARE LICENSE AGREEMENT
#
# Copyright (c) CA, Inc. All rights reserved.
#
# You are hereby granted a non-exclusive, worldwide, royalty-free license
# under CA, Inc.'s copyrights to use, copy, modify, and distribute this
# software in source code or binary form for use in connection with CA, Inc.
# products.
#
# This copyright notice shall be included in all copies or substantial
# portions of the software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
# =============================================================================
function Test-VcfVcenterOverallHealth {

    <#
        .SYNOPSIS
        Checks that every vCenter appliance health endpoint reports healthy and every status endpoint responds.

        .DESCRIPTION
        Queries appliance health endpoints (System, Mem, Storage, Swap, SoftwarePackages, Applmgmt), vCenter services,
        and system uptime across all vCenter appliances connected to SDDC Manager using Get-VcfCheckApplianceHealth,
        Get-VcfCheckVCenterServices, and Get-VcfCheckApplianceUptime.

        Evaluates health states and endpoint responsiveness:
        - Pass: All health endpoints return 'green', and services and system uptime endpoints respond successfully.
        - Fail: One or more health endpoints return 'red'.
        - Warning: One or more health endpoints return 'yellow', 'orange', or 'gray', or an endpoint is unresponsive.

        When a health item is not 'green', queries Get-VcfCheckApplianceHealthMessages to capture localized or default
        diagnostic messages and resolution guidance.

        Populates a structured Rows table detailing Endpoint, Status, and Message.
        Delegates per-vCenter execution and per-domain outcome packaging to Invoke-VcfCheckPerVCenterCheck.

        .PARAMETER Context
        The VcfCheck.Context object. Must already be connected to SDDC Manager.

        .PARAMETER DisplayName
        Optional friendly display name for the check result.

        .OUTPUTS
        [PSObject[]] Per-vCenter check results generated by Invoke-VcfCheckPerVCenterCheck.
    #>


    [CmdletBinding()]
    [OutputType([PSObject])]
    Param (
        [Parameter(Mandatory = $true)] [PSObject]$Context,
        [Parameter(Mandatory = $false)] [String]$DisplayName = ''
    )

    return Invoke-VcfCheckPerVCenterCheck -Context $Context -CheckId 'vcenter_overall_health' -Area vCenter -DisplayName $DisplayName -Body {
        param($Context, $VCenterFqdn)

        $healthItems = @('System', 'Mem', 'Storage', 'Swap', 'SoftwarePackages', 'Applmgmt')
        $unresponsiveItems = [System.Collections.Generic.List[String]]::new()
        $unhealthyItems = [System.Collections.Generic.List[String]]::new()
        $hasRedHealth = $false
        $rows = @()

        foreach ($item in $healthItems) {
            $healthMessage = $null
            try {
                $healthValue = Get-VcfCheckApplianceHealth -Server $VCenterFqdn -Item $item
                $status = switch -Regex ($healthValue) {
                    '^green$' { 'Pass' }
                    '^red$' { 'Fail' }
                    default { 'Warning' }
                }
                if ($status -ne 'Pass') {
                    try {
                        $notifications = Get-VcfCheckApplianceHealthMessages -Server $VCenterFqdn -Item $item
                        $healthMessage = ($notifications | ForEach-Object {
                            $text = if ([String]::IsNullOrEmpty($_.Message.Localized)) { $_.Message.DefaultMessage } else { $_.Message.Localized }
                            $resolutionText = if ([String]::IsNullOrEmpty($_.Resolution.Localized)) { $_.Resolution.DefaultMessage } else { $_.Resolution.Localized }
                            if ($resolutionText) { $text = "$text $resolutionText" }
                            $text
                        }) -join ' '
                    } catch {
                        $healthMessage = $null
                    }
                    $detailSuffix = if ([String]::IsNullOrEmpty($healthMessage)) { '' } else { " ($healthMessage)" }
                    $unhealthyItems.Add("health/$item=$status$detailSuffix")
                    if ($status -eq 'Fail') { $hasRedHealth = $true }
                }
            } catch {
                $healthValue = 'unresponsive'
                $status = 'Warning'
                $unresponsiveItems.Add("health/$item")
            }
            $rows += [PSCustomObject]@{
                Endpoint = "health/$item"
                Status = $status
                Message = if ([String]::IsNullOrEmpty($healthMessage)) { 'N/A' } else { $healthMessage }
            }
        }

        $serviceStatus = 'Pass'
        try {
            Get-VcfCheckVCenterServices -Server $VCenterFqdn | Out-Null
        } catch {
            $unresponsiveItems.Add('services')
            $serviceStatus = 'Warning'
        }
        $rows += [PSCustomObject]@{
            Endpoint = 'services'
            Status = $serviceStatus
            Message = 'N/A'
        }

        $uptimeStatus = 'Pass'
        try {
            Get-VcfCheckApplianceUptime -Server $VCenterFqdn | Out-Null
        } catch {
            $unresponsiveItems.Add('system/uptime')
            $uptimeStatus = 'Warning'
        }
        $rows += [PSCustomObject]@{
            Endpoint = 'system/uptime'
            Status = $uptimeStatus
            Message = 'N/A'
        }

        $detailParts = @()
        if ($unhealthyItems.Count -gt 0) { $detailParts += "Unhealthy: $($unhealthyItems -join '; ')" }
        if ($unresponsiveItems.Count -gt 0) { $detailParts += "Unresponsive: $($unresponsiveItems -join '; ')" }

        if ($hasRedHealth) {
            return [PSCustomObject]@{ Status = 'Fail'; Detail = $detailParts -join ' | '; Rows = $rows }
        } elseif ($detailParts.Count -gt 0) {
            return [PSCustomObject]@{ Status = 'Warning'; Detail = $detailParts -join ' | '; Rows = $rows }
        }
        return [PSCustomObject]@{ Status = 'Pass'; Detail = 'All health/status endpoints reported healthy.'; Rows = $rows }
    }
}