Private/NutanixPrismCentral.ps1
|
<# Prism Central v4 collection cycle. clustermgmt supplies cluster/host inventory and per-host history; vmm supplies VM inventory and VM history through one paginated list endpoint. Host history is fetched with bounded concurrency (Private/Nutanix.ps1), VM history is aggregated per host, and both are joined by sample timestamp into the platform hypervisor payload (Classes/HypervisorPayload.ps1). Reference: https://developers.nutanix.com/api-reference?namespace=clustermgmt&version=v4.0 https://developers.nutanix.com/api-reference?namespace=vmm&version=v4.0 #> # ---------------------------- # Unit conversions # ---------------------------- function ConvertFrom-NutanixPpmToHundredthsOfPercent { param([AllowNull()][object]$Value) if ($null -eq $Value) { return $null } # The Nexthink hypervisor-processor divides usage_average by 100 before storing it, because # vSphere's cpu.usage.average / mem.usage.average counters arrive as hundredths of a percent # ("2938" = 29.38%). Sending a plain percentage here shows up 100x too small in the product. # 1 ppm = 0.0001%, so ppm / 100 is already the hundredths-of-a-percent figure. return [Math]::Round(([double]$Value / 100), 2) } function ConvertFrom-NutanixBytesToMegabytes { param([AllowNull()][object]$Value) if ($null -eq $Value) { return $null } # memory.committed is defined in megabytes (the vSphere connector sums Config.Hardware.MemoryMB) # and the hypervisor-processor multiplies it by 1024^2 on the way to the data model. return [Math]::Round(([double]$Value / 1048576), 0) } function ConvertFrom-NutanixPpmToMilliseconds { param([AllowNull()][object]$Value, [int]$IntervalSeconds) if ($null -eq $Value) { return $null } # Nexthink's Hypervisor API deserializes cpu.ready_summation as a Long; a fractional # millisecond value here fails with a 400 (java.lang.Long from a decimal string). return [Math]::Round(([double]$Value / 1000000) * $IntervalSeconds * 1000, 0) } function ConvertFrom-NutanixMicrosecondsToMilliseconds { param([AllowNull()][object]$Value) if ($null -eq $Value) { return $null } # Nexthink's Hypervisor API deserializes disk.max_total_latency_latest as a Long; see # ConvertFrom-NutanixPpmToMilliseconds for why this must be a whole number. return [Math]::Round(([double]$Value / 1000), 0) } function ConvertTo-NutanixWholeNumber { <# .SYNOPSIS Rounds a counter to a whole number for the payload fields the Hypervisor API deserializes as Long (byte counts and kBps rates). Averaging two 30s points otherwise yields '690.5', which the API rejects with a bare 400. #> param([AllowNull()][object]$Value) if ($null -eq $Value -or [string]::IsNullOrWhiteSpace([string]$Value)) { return $null } $parsed = 0.0 if (-not [double]::TryParse([string]$Value, [System.Globalization.NumberStyles]::Float, [System.Globalization.CultureInfo]::InvariantCulture, [ref]$parsed)) { return $null } return [Math]::Round($parsed, 0) } function Get-NutanixMemoryState { <# .SYNOPSIS Derives the vSphere-style memory pressure state (0=high, 1=soft, 2=hard, 3=low) for an AHV host. .DESCRIPTION AHV has no mem.state counter, so the state follows the "Metrics for Hypervisors" spec: low when less than 1% of host memory is free, hard when VM memory is being swapped out, high otherwise. Soft (ballooning) is never reported: the balloon series (memoryReclaimedByBalloonBytes) and the host-level swap series are rejected with HTTP 500 by pc.7.x, so swap-out comes from the VM tuples. Plain memory usage is deliberately not a pressure signal: hypervisors cache aggressively and ~90% usage is normal, not contention. Without a usage counter there is no positive evidence of a healthy host, so the state stays unset rather than defaulting to high. #> param( [AllowNull()][object]$UsagePpm, [AllowNull()][object]$SwapOutRateKbps ) if ($null -ne $UsagePpm -and ([double]$UsagePpm / 10000) -ge 99) { return 3 } if ($null -ne $SwapOutRateKbps -and [double]$SwapOutRateKbps -gt 0) { return 2 } if ($null -eq $UsagePpm) { return $null } return 0 } function Get-NutanixNestedValue { <# .SYNOPSIS Null-safe read of a nested property path such as 'host.extId' on an API record. #> param( [AllowNull()][object]$Record, [Parameter(Mandatory = $true)][string]$Path ) $current = $Record foreach ($segment in $Path -split '\.') { if ($null -eq $current) { return $null } $property = $current.PSObject.Properties[$segment] if ($null -eq $property) { return $null } $current = $property.Value } return $current } # ---------------------------- # Cluster filtering # ---------------------------- function Test-NutanixClusterHasHosts { <# .SYNOPSIS False for the Prism Central self-registration, which the host list API rejects (CLU-10008). Clusters without a readable clusterFunction are assumed to be AOS clusters. #> param([AllowNull()][object]$Cluster) $functions = @(Get-NutanixNestedValue -Record $Cluster -Path 'config.clusterFunction') foreach ($function in $functions) { if ([string]$function -in $script:NUTANIX_CLUSTER_FUNCTION_SKIP) { return $false } } return $true } # ---------------------------- # VM inventory and history (vmm namespace) # ---------------------------- function Get-NutanixVms { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [pscustomobject]$Context ) Write-CustomLog -Message "Retrieving VMs from Prism Central" -Severity 'DEBUG' $extraQuery = @{} if (-not [string]::IsNullOrWhiteSpace($script:NUTANIX_VM_CONFIG_SELECT)) { $extraQuery['$select'] = $script:NUTANIX_VM_CONFIG_SELECT } $vms = @(Get-NutanixPaginatedResults -Context $Context -Path 'vmm/v4.0/ahv/config/vms' -ExtraQuery $extraQuery) Write-CustomLog -Message "Found $($vms.Count) VMs" -Severity 'DEBUG' return $vms } function Get-NutanixVmsByHostExtId { param([AllowEmptyCollection()][object[]]$Vms) $vmsByHostExtId = @{} foreach ($vm in @($Vms)) { # number_of_vms / number_of_vcpus / committed are defined over powered-on VMs only (the vSphere # connector filters Runtime.PowerState = poweredOn). A powered-off VM normally carries no host # reference, but filter explicitly on powerState too; a VM without powerState is kept. $powerState = [string]$vm.powerState if (-not [string]::IsNullOrWhiteSpace($powerState) -and $powerState -ne 'ON') { continue } $hostExtId = [string](Get-NutanixNestedValue -Record $vm -Path 'host.extId') if ([string]::IsNullOrWhiteSpace($hostExtId)) { continue } if (-not $vmsByHostExtId.ContainsKey($hostExtId)) { $vmsByHostExtId[$hostExtId] = [System.Collections.Generic.List[object]]::new() } $vmsByHostExtId[$hostExtId].Add($vm) } return $vmsByHostExtId } function Get-NutanixVmStatsByExtId { <# .SYNOPSIS Fetches one window of VM history for every VM through the paginated list endpoint and returns a hashtable of VM extId -> stats tuples. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)] [pscustomobject]$Context, [Parameter(Mandatory = $true)] [datetime]$StartUtc, [Parameter(Mandatory = $true)] [datetime]$EndUtc ) $extraQuery = @{ '$startTime' = $StartUtc.ToString('o') '$endTime' = $EndUtc.ToString('o') '$samplingInterval' = $script:NUTANIX_STATS_SAMPLING_INTERVAL_SECONDS # Mandatory on the list endpoint (Prism Central answers 400 VMM-30102 without it), unlike the # per-host stats endpoint where it is optional. '$statType' = $script:NUTANIX_VM_STATS_TYPE } if (-not [string]::IsNullOrWhiteSpace($script:NUTANIX_VM_STATS_SELECT)) { $extraQuery['$select'] = $script:NUTANIX_VM_STATS_SELECT } $results = @(Get-NutanixPaginatedResults -Context $Context -Path 'vmm/v4.0/ahv/stats/vms' -ExtraQuery $extraQuery) $statsByExtId = @{} foreach ($item in $results) { $extId = [string]$item.extId if ([string]::IsNullOrWhiteSpace($extId)) { continue } $statsByExtId[$extId] = @($item.stats) } Write-CustomLog -Message "Retrieved VM history for $($statsByExtId.Count) VM(s) between $($StartUtc.ToString('o')) and $($EndUtc.ToString('o'))" -Severity 'DEBUG' return $statsByExtId } function New-NutanixVmMetricTotals { return [pscustomobject]@{ Ready = 0.0; ReadyAvailable = $false SwapIn = 0.0; SwapInAvailable = $false SwapOut = 0.0; SwapOutAvailable = $false } } function Add-NutanixVmMetricTotal { param( [Parameter(Mandatory = $true)][pscustomobject]$Totals, [Parameter(Mandatory = $true)][object]$Tuple, [Parameter(Mandatory = $true)][string]$FieldName, [Parameter(Mandatory = $true)][string]$TotalName, [Parameter(Mandatory = $false)][double]$Weight = 1.0 ) $property = $Tuple.PSObject.Properties[$FieldName] if ($null -eq $property -or $null -eq $property.Value) { return } $parsed = 0.0 if (-not [double]::TryParse([string]$property.Value, [ref]$parsed)) { return } $Totals.$TotalName += $parsed * $Weight $Totals."${TotalName}Available" = $true } function Get-NutanixVmMetricTotalsByTimestamp { <# .SYNOPSIS Sums the VM-only counters of the given VMs per sample timestamp, so each host event can carry the aggregate of its VMs at the same instant. Keys are the normalized timestamp strings Get-NutanixSampleTimestampKey produces. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)] [AllowEmptyCollection()] [object[]]$Vms, [Parameter(Mandatory = $true)] [hashtable]$VmStatsByExtId ) $totalsByTimestamp = @{} foreach ($vm in @($Vms)) { $extId = [string]$vm.extId if (-not $VmStatsByExtId.ContainsKey($extId)) { continue } # CPU ready must land on the vSphere scale the backend and its 5%/15% health thresholds were # built for. ESXi's host cpu.ready.summation adds up the ready time of every vCPU, whereas # hypervisorCpuReadyTimePpm is one ratio per VM already spread over its vCPUs, so a 10-vCPU VM # reports 10x less for the same contention. Weighting the ppm by the VM's vCPU count restores # the summed-across-vCPUs quantity. A VM without a readable vCPU count is weighted 1. $vcpuWeight = [double](Get-NutanixVmVcpuCount -Vm $vm) if ($vcpuWeight -lt 1) { $vcpuWeight = 1.0 } foreach ($tuple in @($VmStatsByExtId[$extId])) { $timestampKey = Get-NutanixSampleTimestampKey -Sample $tuple if ([string]::IsNullOrWhiteSpace($timestampKey)) { continue } if (-not $totalsByTimestamp.ContainsKey($timestampKey)) { $totalsByTimestamp[$timestampKey] = New-NutanixVmMetricTotals } $totals = $totalsByTimestamp[$timestampKey] Add-NutanixVmMetricTotal -Totals $totals -Tuple $tuple -FieldName 'hypervisorCpuReadyTimePpm' -TotalName 'Ready' -Weight $vcpuWeight Add-NutanixVmMetricTotal -Totals $totals -Tuple $tuple -FieldName 'hypervisorSwapInRateKbps' -TotalName 'SwapIn' Add-NutanixVmMetricTotal -Totals $totals -Tuple $tuple -FieldName 'hypervisorSwapOutRateKbps' -TotalName 'SwapOut' } } return $totalsByTimestamp } # ---------------------------- # Payload mapping # ---------------------------- function Get-NutanixVmVcpuCount { param([AllowNull()][object]$Vm) if ($null -eq $Vm -or $null -eq $Vm.numSockets) { return 0 } $coresPerSocket = if ($null -ne $Vm.numCoresPerSocket) { [int]$Vm.numCoresPerSocket } else { 1 } $threadsPerCore = if ($null -ne $Vm.numThreadsPerCore) { [int]$Vm.numThreadsPerCore } else { 1 } return [int]$Vm.numSockets * $coresPerSocket * $threadsPerCore } function ConvertTo-NutanixPrismCentralVmInfo { param([object]$Vm) $vmInfo = [NutanixHypervisorVMInfo]::new() $vmInfo.name = [string]$Vm.name $vmInfo.guest_tools_version = Get-NutanixNestedValue -Record $Vm -Path 'guestTools.version' $vmInfo.resource_pool = $null $vmInfo.cpu_limit = $null $vmInfo.cpu_shares = $null $throttledIops = Get-NutanixNestedValue -Record $Vm -Path 'storageConfig.qosConfig.throttledIops' $vmInfo.disk_io_limit = if ($null -ne $throttledIops) { [long]$throttledIops } else { $null } return $vmInfo } function Test-NutanixHostSampleHasCounters { <# .SYNOPSIS True when the joined host sample carries at least one time-series counter. Prism Central returns memoryCapacityBytes as a single point stamped with the last capacity change, so a sample carrying only that field is not a performance bucket. #> param([AllowNull()][object]$Sample) if ($null -eq $Sample) { return $false } foreach ($fieldName in $script:NUTANIX_HOST_EVENT_SERIES) { $property = $Sample.PSObject.Properties[$fieldName] if ($null -ne $property -and $null -ne $property.Value) { return $true } } return $false } function Get-NutanixLatestHostSampleValue { <# .SYNOPSIS The value of the given field from the most recent sample that carries it, or null. #> param( [AllowEmptyCollection()][object[]]$Samples, [Parameter(Mandatory = $true)][string]$FieldName ) $latest = $null $latestKey = $null foreach ($sample in @($Samples)) { $value = Get-NutanixStatValueOrNull -Sample $sample -FieldName $FieldName if ($null -eq $value) { continue } $key = Get-NutanixSampleTimestampKey -Sample $sample if ($null -eq $latestKey -or [string]::CompareOrdinal($key, $latestKey) -gt 0) { $latest = $value $latestKey = $key } } return $latest } # ---------------------------- # Resampling 30s Prism Central points onto the 20s buckets of complete minutes # ---------------------------- function Get-NutanixAverageNumeric { <# .SYNOPSIS Mean of the numeric values among the inputs; a non-numeric or null input is ignored. #> param([AllowEmptyCollection()][object[]]$Values) $sum = 0.0; $count = 0 foreach ($value in @($Values)) { if ($null -eq $value) { continue } $parsed = 0.0 if ([double]::TryParse([string]$value, [System.Globalization.NumberStyles]::Float, [System.Globalization.CultureInfo]::InvariantCulture, [ref]$parsed)) { $sum += $parsed; $count++ } } if ($count -eq 0) { return $null } return $sum / $count } function New-NutanixBucketSample { <# .SYNOPSIS One 20s bucket sample built from one or two 30s source samples: every counter present on any source is carried, averaged when both carry it. #> param( [Parameter(Mandatory = $true)][string]$TimestampKey, [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Sources ) $bucket = [ordered]@{ timestamp = $TimestampKey } $fieldNames = @($Sources | ForEach-Object { $_.PSObject.Properties.Name } | Where-Object { $_ -ne 'timestamp' } | Select-Object -Unique) foreach ($fieldName in $fieldNames) { $average = Get-NutanixAverageNumeric -Values @($Sources | ForEach-Object { Get-NutanixStatValueOrNull -Sample $_ -FieldName $fieldName }) if ($null -ne $average) { $bucket[$fieldName] = $average } } return [pscustomobject]$bucket } function New-NutanixBucketVmMetricTotals { <# .SYNOPSIS VM aggregate for one 20s bucket from the aggregates of one or two 30s source timestamps. #> param([Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Sources) $totals = New-NutanixVmMetricTotals foreach ($name in @('Ready', 'SwapIn', 'SwapOut')) { $available = @($Sources | Where-Object { $null -ne $_ -and $_."${name}Available" }) if ($available.Count -eq 0) { continue } $totals.$name = ($available | Measure-Object -Property $name -Average).Average $totals."${name}Available" = $true } return $totals } function Get-NutanixCompleteMinuteBuckets { <# .SYNOPSIS Resamples 30s host samples and VM aggregates onto 20s buckets, one complete minute at a time. .DESCRIPTION Starting at StartUtc (minute-aligned), a minute is complete when the host has a point at every 30s offset. Leading incomplete minutes are skipped (a gap Prism Central will never fill); the first incomplete minute after a complete one ends the run, because the trailing minute is the one Prism Central is still filling and it is re-read next cycle. .OUTPUTS Buckets: ordered list of { TimestampKey; Sample; VmTotals }. CoveredEndUtc: end of the last emitted minute, or StartUtc when nothing was emitted. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$HostSamples, [Parameter(Mandatory = $true)][hashtable]$VmTotalsByTimestamp, [Parameter(Mandatory = $true)][datetime]$StartUtc, [Parameter(Mandatory = $true)][datetime]$EndUtc ) $samplesByKey = @{} foreach ($sample in @($HostSamples)) { $key = Get-NutanixSampleTimestampKey -Sample $sample if (-not [string]::IsNullOrWhiteSpace($key)) { $samplesByKey[$key] = $sample } } $resolution = [int]$script:NUTANIX_PC_STATS_RESOLUTION_SECONDS $bucketSeconds = [int]$script:NUTANIX_STATS_SAMPLING_INTERVAL_SECONDS $buckets = [System.Collections.Generic.List[object]]::new() $coveredEnd = $StartUtc $runStarted = $false for ($minuteStart = $StartUtc; $minuteStart.AddSeconds(60) -le $EndUtc; $minuteStart = $minuteStart.AddSeconds(60)) { $sourceKeys = @(for ($offset = 0; $offset -lt 60; $offset += $resolution) { $minuteStart.AddSeconds($offset).ToString('o') }) $complete = @($sourceKeys | Where-Object { -not $samplesByKey.ContainsKey($_) }).Count -eq 0 if (-not $complete) { if ($runStarted) { break } continue } $runStarted = $true for ($index = 0; $index -lt $script:NUTANIX_BUCKET_SOURCE_OFFSETS.Count; $index++) { $offsets = @($script:NUTANIX_BUCKET_SOURCE_OFFSETS[$index]) $sources = @($offsets | ForEach-Object { $samplesByKey[$minuteStart.AddSeconds($_).ToString('o')] }) $totalSources = @($offsets | ForEach-Object { $k = $minuteStart.AddSeconds($_).ToString('o'); if ($VmTotalsByTimestamp.ContainsKey($k)) { $VmTotalsByTimestamp[$k] } }) $bucketKey = $minuteStart.AddSeconds($index * $bucketSeconds).ToString('o') $buckets.Add([pscustomobject]@{ TimestampKey = $bucketKey Sample = New-NutanixBucketSample -TimestampKey $bucketKey -Sources $sources VmTotals = New-NutanixBucketVmMetricTotals -Sources $totalSources }) } $coveredEnd = $minuteStart.AddSeconds(60) } return [pscustomobject]@{ Buckets = @($buckets); CoveredEndUtc = $coveredEnd } } function New-NutanixPrismCentralHostEvent { param( [Parameter(Mandatory = $true)][object]$HostItem, [Parameter(Mandatory = $true)][object]$HostSample, [Parameter(Mandatory = $true)][int]$NumberOfVcpus, [Parameter(Mandatory = $true)][pscustomobject]$VmMetricTotals, [Parameter(Mandatory = $false)][AllowNull()][object]$MemoryCapacityBytes = $null, [Parameter(Mandatory = $false)][AllowNull()][object]$CommittedMemoryBytes = $null ) $intervalSeconds = $script:NUTANIX_STATS_SAMPLING_INTERVAL_SECONDS $cpuPpm = Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'hypervisorCpuUsagePpm' $memoryPpm = Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'aggregateHypervisorMemoryUsagePpm' $hostEvent = [NutanixHypervisorEvent]::new() # A v4 stats tuple is stamped with the start of its sampling bucket, like a v2 series point. $hostEvent.start_time = ConvertTo-Rfc3339UtcZ -Timestamp (ConvertFrom-RfcUtcTimestamp -Value (Get-NutanixSampleTimestampKey -Sample $HostSample)) $hostEvent.duration = $intervalSeconds $cpu = [NutanixHypervisorCpuMetrics]::new() $cpu.number_of_threads = [int]$HostItem.numberOfCpuThreads $cpu.number_of_packages = [int]$HostItem.numberOfCpuSockets $cpu.number_of_vcpus = $NumberOfVcpus $cpu.ready_summation = if ($VmMetricTotals.ReadyAvailable) { ConvertFrom-NutanixPpmToMilliseconds $VmMetricTotals.Ready $intervalSeconds } else { $null } $cpu.usage_average = ConvertFrom-NutanixPpmToHundredthsOfPercent $cpuPpm $cpu.used_summation = if ($null -ne $cpuPpm) { # Nexthink's Hypervisor API deserializes cpu.used_summation as a Long; see # ConvertFrom-NutanixPpmToMilliseconds for why this must be a whole number. [Math]::Round(([double]$cpuPpm / 1000000) * [int]$HostItem.numberOfCpuThreads * $intervalSeconds * 1000, 0) } else { $null } $hostEvent.cpu = $cpu $disk = [NutanixHypervisorDiskMetrics]::new() $disk.read_average = ConvertTo-NutanixWholeNumber (Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'controllerReadIoBandwidthKbps') $disk.write_average = ConvertTo-NutanixWholeNumber (Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'controllerWriteIoBandwidthKbps') $disk.max_total_latency_latest = ConvertFrom-NutanixMicrosecondsToMilliseconds (Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'controllerAvgIoLatencyUsecs') $hostEvent.disk = $disk $memory = [NutanixHypervisorMemoryMetrics]::new() $swapOut = if ($VmMetricTotals.SwapOutAvailable) { ConvertTo-NutanixWholeNumber $VmMetricTotals.SwapOut } else { $null } $memory.swap_in_rate_average = if ($VmMetricTotals.SwapInAvailable) { ConvertTo-NutanixWholeNumber $VmMetricTotals.SwapIn } else { $null } $memory.swap_out_rate_average = $swapOut $memory.swap_used_average = $null $memory.state_latest = Get-NutanixMemoryState -UsagePpm $memoryPpm -SwapOutRateKbps $swapOut $memory.vm_mem_ctl_average = $null $memory.usage_average = ConvertFrom-NutanixPpmToHundredthsOfPercent $memoryPpm $installedMemory = Get-NutanixStatValueOrNull -Sample $HostSample -FieldName 'memoryCapacityBytes' if ($null -eq $installedMemory -and $null -ne $MemoryCapacityBytes) { $installedMemory = [string]$MemoryCapacityBytes } $memory.installed = if ($null -ne $installedMemory) { ConvertTo-NutanixWholeNumber $installedMemory } else { ConvertTo-NutanixWholeNumber $HostItem.memorySizeBytes } # Committed memory is the memory configured for the powered-on VMs (spec: HostStats # totalConfiguredVmMemoryBytes, or the sum of Vm.memorySizeBytes; pc.7.x rejects the former with # HTTP 500, so the VM configs are summed). Sent in megabytes: the backend multiplies by 1024^2. $memory.committed = ConvertFrom-NutanixBytesToMegabytes $CommittedMemoryBytes $hostEvent.memory = $memory return $hostEvent } function ConvertTo-NutanixPrismCentralDataItem { <# .OUTPUTS { DataItem; CoveredEndUtc }. CoveredEndUtc is the end of the last complete minute emitted, which is how far the host's watermark may advance; StartUtc when no minute was complete. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)][object]$HostItem, [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$HostSamples, [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]]$Vms, [Parameter(Mandatory = $true)][hashtable]$VmStatsByExtId, [Parameter(Mandatory = $true)][datetime]$StartUtc, [Parameter(Mandatory = $true)][datetime]$EndUtc ) $numberOfVcpus = 0 $committedMemoryBytes = $null foreach ($vm in @($Vms)) { $numberOfVcpus += Get-NutanixVmVcpuCount -Vm $vm $memorySizeBytes = Get-NutanixNestedValue -Record $vm -Path 'memorySizeBytes' if ($null -ne $memorySizeBytes) { $committedMemoryBytes = [double]$committedMemoryBytes + [double]$memorySizeBytes } } $vmTotalsByTimestamp = Get-NutanixVmMetricTotalsByTimestamp -Vms @($Vms) -VmStatsByExtId $VmStatsByExtId $memoryCapacityBytes = Get-NutanixLatestHostSampleValue -Samples @($HostSamples) -FieldName 'memoryCapacityBytes' $counterSamples = @($HostSamples | Where-Object { Test-NutanixHostSampleHasCounters -Sample $_ }) $resampled = Get-NutanixCompleteMinuteBuckets -HostSamples $counterSamples -VmTotalsByTimestamp $vmTotalsByTimestamp -StartUtc $StartUtc -EndUtc $EndUtc $events = [System.Collections.Generic.List[NutanixHypervisorEvent]]::new() foreach ($bucket in $resampled.Buckets) { $hostEvent = New-NutanixPrismCentralHostEvent -HostItem $HostItem -HostSample $bucket.Sample -NumberOfVcpus $numberOfVcpus ` -VmMetricTotals $bucket.VmTotals -MemoryCapacityBytes $memoryCapacityBytes -CommittedMemoryBytes $committedMemoryBytes $events.Add($hostEvent) } if ($events.Count -eq 0 -and $counterSamples.Count -gt 0) { Write-CustomLog -Message "Host '$($HostItem.hostName)' returned $($counterSamples.Count) sample(s) but no complete minute between $($StartUtc.ToString('o')) and $($EndUtc.ToString('o')); nothing emitted, window retried next cycle." -Severity 'WARNING' } $hostInfo = [NutanixHypervisorHostInfo]::new() $hostInfo.name = [string]$HostItem.hostName $hostInfo.cluster = [string]$HostItem.ClusterName $hostInfo.number_of_vms = @($Vms).Count $hostInfo.power_policy = $null # No AHV equivalent. $hostInfo.hyperthreading = ([int]$HostItem.numberOfCpuThreads -gt [int]$HostItem.numberOfCpuCores) $virtualMachines = [System.Collections.Generic.List[NutanixHypervisorVMInfo]]::new() foreach ($vm in @($Vms)) { $virtualMachines.Add((ConvertTo-NutanixPrismCentralVmInfo -Vm $vm)) } $dataItem = [NutanixHypervisorDataItem]::new() $dataItem.host = $hostInfo $dataItem.events = @($events) $dataItem.virtual_machines = @($virtualMachines) return [pscustomobject]@{ DataItem = $dataItem; CoveredEndUtc = $resampled.CoveredEndUtc } } # ---------------------------- # Cycle orchestrator # ---------------------------- function Invoke-NutanixPrismCentralCollection { [CmdletBinding()] param( [Parameter(Mandatory = $true)][NutanixConnectorConfiguration]$Config, [Parameter(Mandatory = $true)][NutanixConnectorState]$State ) $cycleStopwatch = [Diagnostics.Stopwatch]::StartNew() $context = Get-NutanixApiContext -EnvironmentConfig $Config.EnvironmentConfig $now = [datetime]::UtcNow $summary = [ordered]@{ Clusters = 0; ClusterFailures = 0; Hosts = 0; HostFailures = 0; Vms = 0; VmStatsFailures = 0 } # Inventory $clusters = @(Get-NutanixClusters -Context $context) $summary.Clusters = $clusters.Count $allHosts = [System.Collections.Generic.List[object]]::new() foreach ($cluster in $clusters) { if (-not (Test-NutanixClusterHasHosts -Cluster $cluster)) { Write-CustomLog -Message "Skipping cluster '$($cluster.name)' ($($cluster.extId)): clusterFunction $((@(Get-NutanixNestedValue -Record $cluster -Path 'config.clusterFunction')) -join ',') has no AHV hosts." -Severity 'DEBUG' continue } try { $clusterHosts = @(Get-NutanixHosts -Context $context -ClusterExtId $cluster.extId -ClusterName $cluster.name) } catch { # Prism Central registers itself as a cluster without AHV hosts and rejects the host # listing; one unreadable cluster must not cost the others their cycle. $summary.ClusterFailures++ Write-CustomLog -Message "Skipping cluster '$($cluster.name)' ($($cluster.extId)): host listing failed. Error=$($_.Exception.Message)" -Severity 'WARNING' continue } foreach ($hostItem in $clusterHosts) { $allHosts.Add($hostItem) } } $summary.Hosts = $allHosts.Count $vms = @() if ($allHosts.Count -gt 0) { $vms = @(Get-NutanixVms -Context $context) } $summary.Vms = $vms.Count $vmsByHostExtId = Get-NutanixVmsByHostExtId -Vms $vms # Per-host history windows $fallbackWatermark = ConvertFrom-RfcUtcTimestamp -Value $State.watermarks.last_received_utc $workItems = [System.Collections.Generic.List[object]]::new() foreach ($hostItem in $allHosts) { $hostExtId = [string]$hostItem.extId $lastReceivedUtc = $fallbackWatermark if ($State.watermarks.host_last_received_utc.ContainsKey($hostExtId)) { $lastReceivedUtc = ConvertFrom-RfcUtcTimestamp -Value $State.watermarks.host_last_received_utc[$hostExtId] } $range = Get-HypervisorMetricsTimeRange -Now $now -LastReceivedUtc $lastReceivedUtc -MaxMinutesRead $script:MAX_MINUTES_READ if ($range.End -le $range.Start) { continue } $workItems.Add((New-NutanixHostStatsWorkItem -Host $hostItem -StartUtc $range.Start -EndUtc $range.End)) } $rangeContext = $null if ($workItems.Count -gt 0) { $rangeContext = [pscustomobject]@{ Start = @($workItems | ForEach-Object { $_.StartUtc }) | Sort-Object | Select-Object -First 1 End = @($workItems | ForEach-Object { $_.EndUtc }) | Sort-Object -Descending | Select-Object -First 1 } } # History $hostStats = Get-NutanixHostStatsConcurrent -Context $context -WorkItems @($workItems) $summary.HostFailures = @($hostStats.FailedWorkItems).Count $vmStatsByExtId = @{} if ($null -ne $rangeContext -and $vms.Count -gt 0) { try { $vmStatsByExtId = Get-NutanixVmStatsByExtId -Context $context -StartUtc $rangeContext.Start -EndUtc $rangeContext.End } catch { $summary.VmStatsFailures = 1 Write-CustomLog -Message "VM history collection failed; host events carry no VM-derived counters this cycle. Error=$($_.Exception.Message)" -Severity 'ERROR' } } # Assembly $dataItems = [System.Collections.Generic.List[NutanixHypervisorDataItem]]::new() $coveredEnds = [System.Collections.Generic.List[datetime]]::new() foreach ($workItem in $workItems) { $hostExtId = [string]$workItem.Host.extId if (-not $hostStats.SamplesByHostExtId.ContainsKey($hostExtId)) { # Failed host: keep its previous watermark so the window is retried next cycle. continue } $hostVms = @() if ($vmsByHostExtId.ContainsKey($hostExtId)) { $hostVms = @($vmsByHostExtId[$hostExtId]) } $assembled = ConvertTo-NutanixPrismCentralDataItem -HostItem $workItem.Host ` -HostSamples @($hostStats.SamplesByHostExtId[$hostExtId]) -Vms $hostVms -VmStatsByExtId $vmStatsByExtId ` -StartUtc $workItem.StartUtc -EndUtc $workItem.EndUtc $dataItems.Add($assembled.DataItem) # Only the complete minutes actually emitted count as received; the trailing minute Prism # Central had not finished stays after the watermark and is read again next cycle. if ($assembled.CoveredEndUtc -gt $workItem.StartUtc) { $coveredEnds.Add($assembled.CoveredEndUtc) $State.watermarks.host_last_received_utc[$hostExtId] = $assembled.CoveredEndUtc.ToString('o') } } if ($coveredEnds.Count -gt 0) { # The spool timestamp and reported range end reflect what was actually emitted. $rangeContext.End = @($coveredEnds) | Sort-Object -Descending | Select-Object -First 1 } $payload = [NutanixHypervisorPayload]::new() $payload.schema_version = '1.1' $payload.source = 'nutanix-connector' $payload.customer_environment = $Config.EnvironmentConfig.Name $payload.version = Get-ModuleVersion $payload.data = @($dataItems) if ($dataItems.Count -gt 0 -and $null -ne $rangeContext) { $null = Write-SpoolReceived -Config $Config -Timestamp ([DateTimeOffset]$rangeContext.End) -Content $payload } $hostWatermarks = foreach ($hostItem in $allHosts) { $hostExtId = [string]$hostItem.extId if ($State.watermarks.host_last_received_utc.ContainsKey($hostExtId)) { ConvertFrom-RfcUtcTimestamp -Value $State.watermarks.host_last_received_utc[$hostExtId] } else { $fallbackWatermark } } if (@($hostWatermarks).Count -gt 0) { $State.watermarks.last_received_utc = (@($hostWatermarks) | Sort-Object | Select-Object -First 1).ToString('o') } Save-State -State $State -Config $Config $cycleStopwatch.Stop() $summary.StartUtc = if ($null -ne $rangeContext) { $rangeContext.Start.ToString('o') } else { $null } $summary.EndUtc = if ($null -ne $rangeContext) { $rangeContext.End.ToString('o') } else { $null } $summary.DurationSeconds = [Math]::Round($cycleStopwatch.Elapsed.TotalSeconds, 2) return [pscustomobject]@{ Payload = $payload Range = $rangeContext Summary = [pscustomobject]$summary } } |