Private/Get-vSphereHypervisorMetrics.ps1
|
# ---------------------------- # Phase 1: Retrieve ESXi hosts, clusters, and VM counts # ---------------------------- function Get-VSphereHostsData { [CmdletBinding()] param() Write-CustomLog -Message "Retrieving ESXi hosts from vSphere using Get-View" -Severity 'DEBUG' $hostViews = @(Get-View -ViewType HostSystem -Property Name, Parent, Hardware.SystemInfo.Uuid, Hardware.CpuInfo.NumCpuPackages, Summary.Hardware.NumCpuThreads, Config.PowerSystemInfo.CurrentPolicy.Key, Config.HyperThread.Active, Vm -ErrorAction Stop) Write-CustomLog -Message "Found $($hostViews.Count) ESXi hosts" -Severity 'DEBUG' Write-CustomLog -Message "Retrieving cluster information" -Severity 'DEBUG' $clusterNameByMoRef = @{} $clusterViews = @(Get-View -ViewType ClusterComputeResource -Property Name -ErrorAction SilentlyContinue) foreach ($clusterView in $clusterViews) { if ($null -eq $clusterView -or $null -eq $clusterView.MoRef -or $null -eq $clusterView.MoRef.Value) { continue } $clusterNameByMoRef[$clusterView.MoRef.Value] = $clusterView.Name } Write-CustomLog -Message "Retrieving VM count and vCPU count per host" -Severity 'DEBUG' $vmCountByHostMoRef = @{} $vcpuCountByHostMoRef = @{} $vmViews = @(Get-View -ViewType VirtualMachine -Property Runtime.Host, Config.Hardware.NumCPU -Filter @{ 'Runtime.PowerState' = 'poweredOn' } -ErrorAction SilentlyContinue) foreach ($vm in $vmViews) { if ($null -eq $vm.Runtime -or $null -eq $vm.Runtime.Host) { continue } $hostMoRef = $vm.Runtime.Host.Value if (-not $vmCountByHostMoRef.ContainsKey($hostMoRef)) { $vmCountByHostMoRef[$hostMoRef] = 0 } $vmCountByHostMoRef[$hostMoRef]++ $numCpu = if ($null -ne $vm.Config -and $null -ne $vm.Config.Hardware) { [int]$vm.Config.Hardware.NumCPU } else { 0 } if (-not $vcpuCountByHostMoRef.ContainsKey($hostMoRef)) { $vcpuCountByHostMoRef[$hostMoRef] = 0 } $vcpuCountByHostMoRef[$hostMoRef] += $numCpu } return [pscustomobject]@{ HostViews = $hostViews ClusterNameByMoRef = $clusterNameByMoRef VMCountByHostMoRef = $vmCountByHostMoRef VCPUCountByHostMoRef = $vcpuCountByHostMoRef } } # ---------------------------- # Phase 2a: Set up PerfManager and counter mappings # ---------------------------- function Initialize-PerfCounterMappings { [CmdletBinding()] param( [Parameter(Mandatory = $true)] $PerfManager ) $counterIdByName = @{} if ($null -ne $PerfManager.PerfCounter) { foreach ($counter in $PerfManager.PerfCounter) { if ($null -eq $counter -or $null -eq $counter.Key) { continue } $groupKey = if ($null -ne $counter.GroupInfo) { $counter.GroupInfo.Key } else { '' } $nameKey = if ($null -ne $counter.NameInfo) { $counter.NameInfo.Key } else { '' } $rollupType = if ($null -ne $counter.RollupType) { $counter.RollupType } else { '' } $metricName = "{0}.{1}.{2}" -f $groupKey, $nameKey, $rollupType if (-not $counterIdByName.ContainsKey($metricName)) { $counterIdByName[$metricName] = $counter.Key } } } $requestedCounterIds = @() foreach ($metricName in $script:VSPHERE_HYPERVISOR_METRIC_IDS) { if (-not $counterIdByName.ContainsKey($metricName)) { Write-CustomLog -Message "Counter not found in vCenter: $metricName" -Severity 'WARNING' continue } $requestedCounterIds += [pscustomobject]@{ Name = $metricName Id = [int]$counterIdByName[$metricName] } } $counterNameById = @{} foreach ($counterInfo in $requestedCounterIds) { $counterNameById[$counterInfo.Id] = $counterInfo.Name } return [pscustomobject]@{ RequestedCounterIds = $requestedCounterIds CounterNameById = $counterNameById } } # ---------------------------- # Phase 2b: Build host data structures and query metrics in batches # ---------------------------- function Get-HostPerformanceMetrics { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [array]$HostViews, [Parameter(Mandatory = $true)] $PerfManager, [Parameter(Mandatory = $true)] [pscustomobject]$CounterMappings, [Parameter(Mandatory = $true)] [hashtable]$ClusterNameByMoRef, [Parameter(Mandatory = $true)] [hashtable]$VMCountByHostMoRef, [Parameter(Mandatory = $true)] [hashtable]$VCPUCountByHostMoRef, [Parameter(Mandatory = $true)] [datetime]$StartUtc, [Parameter(Mandatory = $true)] [datetime]$EndUtc ) # Helper: Split array into batches function Split-ArrayIntoBatches { param( [Parameter(Mandatory = $true)] [array]$InputArray, [Parameter(Mandatory = $true)] [int]$Size ) for ($batchStartIndex = 0; $batchStartIndex -lt $InputArray.Count; $batchStartIndex += $Size) { , $InputArray[$batchStartIndex..([Math]::Min($batchStartIndex + $Size - 1, $InputArray.Count - 1))] } } $hostDataByMoRef = @{} foreach ($hostView in $HostViews) { if ($null -eq $hostView -or $null -eq $hostView.MoRef) { continue } $hostMoRef = $hostView.MoRef.Value if ($null -eq $hostMoRef) { continue } $clusterName = $null if ($null -ne $hostView.Parent -and $hostView.Parent.Type -eq 'ClusterComputeResource') { $clusterName = $ClusterNameByMoRef[$hostView.Parent.Value] } $vmCount = 0 if ($VMCountByHostMoRef.ContainsKey($hostMoRef)) { $vmCount = $VMCountByHostMoRef[$hostMoRef] } $vcpuCount = 0 if ($VCPUCountByHostMoRef.ContainsKey($hostMoRef)) { $vcpuCount = $VCPUCountByHostMoRef[$hostMoRef] } $hostDataByMoRef[$hostMoRef] = [pscustomobject]@{ HostView = $hostView Name = $hostView.Name Cluster = $clusterName VMCount = $vmCount VCPUCount = $vcpuCount EventsByTimestamp = @{} } } Write-CustomLog -Message "Querying performance metrics for $($HostViews.Count) hosts in batches of $($script:PERF_QUERY_BATCH_SIZE)" -Severity 'DEBUG' $intervalId = $script:METRICS_INTERVAL_SECONDS $requestedCounterIds = $CounterMappings.RequestedCounterIds $counterNameById = $CounterMappings.CounterNameById $metricIds = foreach ($counterInfo in $requestedCounterIds) { $perfMetricId = New-Object VMware.Vim.PerfMetricId $perfMetricId.CounterId = $counterInfo.Id $perfMetricId.Instance = "" $perfMetricId } foreach ($batch in (Split-ArrayIntoBatches -InputArray $HostViews -Size $script:PERF_QUERY_BATCH_SIZE)) { $specs = foreach ($hostView in $batch) { $querySpec = New-Object VMware.Vim.PerfQuerySpec $querySpec.Entity = $hostView.MoRef $querySpec.IntervalId = $intervalId $querySpec.StartTime = $StartUtc $querySpec.EndTime = $EndUtc $querySpec.MetricId = $metricIds $querySpec } $results = $PerfManager.QueryPerf($specs) foreach ($perfResult in $results) { $hostMoRef = $perfResult.Entity.Value if (-not $hostDataByMoRef.ContainsKey($hostMoRef)) { continue } $hostData = $hostDataByMoRef[$hostMoRef] if ($null -eq $perfResult.SampleInfo) { continue } for ($sampleIndex = 0; $sampleIndex -lt $perfResult.SampleInfo.Count; $sampleIndex++) { $timestamp = $perfResult.SampleInfo[$sampleIndex].Timestamp $timestampKey = $timestamp.ToString('o') if (-not $hostData.EventsByTimestamp.ContainsKey($timestampKey)) { $hostData.EventsByTimestamp[$timestampKey] = @{ Timestamp = $timestamp Metrics = @{} } } foreach ($series in $perfResult.Value) { $counterId = [int]$series.Id.CounterId if (-not $counterNameById.ContainsKey($counterId)) { continue } $metricName = $counterNameById[$counterId] $value = $series.Value[$sampleIndex] $hostData.EventsByTimestamp[$timestampKey].Metrics[$metricName] = $value } } } } return $hostDataByMoRef } # ---------------------------- # Phase 3: Build output data items # ---------------------------- function ConvertTo-HypervisorDataItems { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [hashtable]$HostDataByMoRef ) # Helper: Get metric value as string or null from stats hashtable function Get-MetricValueOrNull { param( [Parameter(Mandatory = $true)] [hashtable]$StatsByMetricName, [Parameter(Mandatory = $true)] [string]$MetricName ) if ($StatsByMetricName.ContainsKey($MetricName)) { return [string]$StatsByMetricName[$MetricName] } return $null } Write-CustomLog -Message "Building output data items" -Severity 'DEBUG' $dataItems = foreach ($hostMoRef in $HostDataByMoRef.Keys) { $hostData = $HostDataByMoRef[$hostMoRef] $hostInfo = [VSphereHypervisorHostInfo]::new() $hostInfo.name = [string]$hostData.Name $hostInfo.cluster = $hostData.Cluster $hostInfo.number_of_vms = [int]$hostData.VMCount $hostInfo.power_policy = $null if ($null -ne $hostData.HostView.Config -and $null -ne $hostData.HostView.Config.PowerSystemInfo -and $null -ne $hostData.HostView.Config.PowerSystemInfo.CurrentPolicy) { $hostInfo.power_policy = [int]$hostData.HostView.Config.PowerSystemInfo.CurrentPolicy.Key } $hostInfo.hyperthreading = $false if ($null -ne $hostData.HostView.Config -and $null -ne $hostData.HostView.Config.HyperThread) { $hostInfo.hyperthreading = [bool]$hostData.HostView.Config.HyperThread.Active } $events = @() $sortedTimestamps = $hostData.EventsByTimestamp.Keys | Sort-Object foreach ($timestampKey in $sortedTimestamps) { $eventData = $hostData.EventsByTimestamp[$timestampKey] $eventTimestamp = $eventData.Timestamp $metrics = $eventData.Metrics $hypervisorEvent = [VSphereHypervisorEvent]::new() $hypervisorEvent.start_time = ConvertTo-Rfc3339UtcZ -Timestamp $eventTimestamp $hypervisorEvent.duration = [int]$script:METRICS_INTERVAL_SECONDS $hypervisorEvent.cpu = [VSphereHypervisorCpuMetrics]::new() $hypervisorEvent.disk = [VSphereHypervisorDiskMetrics]::new() $hypervisorEvent.memory = [VSphereHypervisorMemoryMetrics]::new() # CPU $hypervisorEvent.cpu.number_of_threads = [int]$hostData.HostView.Summary.Hardware.NumCpuThreads $hypervisorEvent.cpu.number_of_packages = [int]$hostData.HostView.Hardware.CpuInfo.NumCpuPackages $hypervisorEvent.cpu.number_of_vcpus = [int]$hostData.VCPUCount $hypervisorEvent.cpu.ready_summation = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.CpuReadySummation $hypervisorEvent.cpu.usage_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.CpuUsageAverage $hypervisorEvent.cpu.used_summation = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.CpuUsedSummation # Disk $hypervisorEvent.disk.read_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.DiskReadAverage $hypervisorEvent.disk.write_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.DiskWriteAverage $hypervisorEvent.disk.max_total_latency_latest = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.DiskMaxTotalLatencyLatest # Memory $hypervisorEvent.memory.swap_in_rate_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemSwapInRateAverage $hypervisorEvent.memory.swap_out_rate_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemSwapOutRateAverage $hypervisorEvent.memory.swap_used_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemSwapUsedAverage $hypervisorEvent.memory.state_latest = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemStateLatest $hypervisorEvent.memory.vm_mem_ctl_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemVmmemctlAverage $hypervisorEvent.memory.usage_average = Get-MetricValueOrNull -StatsByMetricName $metrics -MetricName $script:VSPHERE_HYPERVISOR_METRICS_AGGREGATED.MemUsageAverage $events += $hypervisorEvent } $dataItem = [VSphereHypervisorDataItem]::new() $dataItem.host = $hostInfo $dataItem.events = $events $dataItem } return $dataItems } # ---------------------------- # Main orchestrator function # ---------------------------- function Get-vSphereHypervisorMetrics { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [VSphereEnvironmentConfiguration]$EnvironmentConfig, [Parameter(Mandatory = $true)] [datetime]$Start, [Parameter(Mandatory = $true)] [datetime]$End ) $connection = $null try { Write-CustomLog -Message "Connecting to vSphere server: $($EnvironmentConfig.vCenterFQDN)" -Severity 'DEBUG' $connection = Connect-VSphereServer -Server $EnvironmentConfig.vCenterFQDN -Target $EnvironmentConfig.WindowsCredentialEntry $startUtc = $Start.ToUniversalTime() $endUtc = $End.ToUniversalTime() if ($endUtc -le $startUtc) { throw "Invalid time window: End must be after Start. Start='$startUtc' End='$endUtc'" } # Phase 1: Retrieve hosts, clusters, and VM data $hostsData = Get-VSphereHostsData if ($hostsData.HostViews.Count -eq 0) { return @() } # Phase 2: Setup PerfManager and counter mappings Write-CustomLog -Message "Setting up Performance Manager" -Severity 'DEBUG' $serviceInstance = Get-View ServiceInstance -ErrorAction Stop $perfMgr = Get-View $serviceInstance.Content.PerfManager -ErrorAction Stop $counterMappings = Initialize-PerfCounterMappings -PerfManager $perfMgr # Phase 2: Query performance metrics $hostDataByMoRef = Get-HostPerformanceMetrics ` -HostViews $hostsData.HostViews ` -PerfManager $perfMgr ` -CounterMappings $counterMappings ` -ClusterNameByMoRef $hostsData.ClusterNameByMoRef ` -VMCountByHostMoRef $hostsData.VMCountByHostMoRef ` -VCPUCountByHostMoRef $hostsData.VCPUCountByHostMoRef ` -StartUtc $startUtc ` -EndUtc $endUtc # Phase 3: Build output data items return ConvertTo-HypervisorDataItems -HostDataByMoRef $hostDataByMoRef } catch { throw "Failed to retrieve vSphere hypervisor metrics: $_" } finally { Disconnect-VSphereServer -Connection $connection } } function Get-HypervisorMetricsPayload { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [VSphereConnectorState]$State, [Parameter(Mandatory = $true)] [VSphereConnectorConfiguration]$Config ) $now = [DateTime]::UtcNow # Parse last_received_utc from state $lastReceivedUtc = ConvertFrom-RfcUtcTimestamp -Value $State.watermarks.last_received_utc $range = Get-HypervisorMetricsTimeRange -Now $now -LastReceivedUtc $lastReceivedUtc -MaxMinutesRead $script:MAX_MINUTES_READ $start = $range.Start $end = $range.End Write-CustomLog -Message "Fetching hypervisor metrics from $start to $end" -Severity 'INFO' # Fetch metrics $dataItems = @(Get-vSphereHypervisorMetrics -EnvironmentConfig $Config.EnvironmentConfig -Start $start -End $end) # Build payload $payload = [VSphereHypervisorPayload]::new() $payload.schema_version = '1.0' $payload.source = 'vsphere-connector' $payload.customer_environment = $Config.EnvironmentConfig.Name $payload.version = Get-ModuleVersion $payload.data = $dataItems # Save payload to spool (timestamp aligned with fetched time window) # In PowerShell, any uncaptured function return values automatically become part of the calling function's output stream. $null = Write-SpoolReceived -Config $Config -Timestamp ([DateTimeOffset]$end) -Content $payload # Update state watermark $State.watermarks.last_received_utc = $end.ToString('o') Save-State -State $State -Config $Config Write-CustomLog -Message "Retrieved metrics for $($dataItems.Count) hosts. Updated last_received_utc to $($end.ToString('o'))" -Severity 'INFO' return @{Payload = $payload ; Range = $range} } function Get-HypervisorMetricsTimeRange { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [datetime]$Now, [Parameter(Mandatory = $true)] [datetime]$LastReceivedUtc, [Parameter(Mandatory = $true)] [int]$MaxMinutesRead ) # If last_received is more than 1 hour in the past, move start to 55 minutes ago $oneHourAgo = $Now.AddHours(-1) if ($LastReceivedUtc -lt $oneHourAgo) { Write-CustomLog -Message "Last received timestamp ($LastReceivedUtc) is more than 1 hour ago. Moving start to 55 minutes ago." -Severity 'INFO' $start = $Now.AddMinutes(-55) } else { $start = $LastReceivedUtc } # Round start to minute (floor/truncate) $start = [datetime]::new($start.Year, $start.Month, $start.Day, $start.Hour, $start.Minute, 0, [System.DateTimeKind]::Utc) # End = min(now - 1 minute, start + MaxMinutesRead) $nowMinus1 = $Now.AddMinutes(-1) $startPlusMax = $start.AddMinutes($MaxMinutesRead) $end = if ($nowMinus1 -lt $startPlusMax) { $nowMinus1 } else { $startPlusMax } # Round end to minute (floor/truncate) $end = [datetime]::new($end.Year, $end.Month, $end.Day, $end.Hour, $end.Minute, 0, [System.DateTimeKind]::Utc) return [pscustomobject]@{ Start = $start End = $end } } |