Private/Nutanix.ps1
|
<# Prism Central v4 inventory and host-history primitives. Cluster and host inventory come from the clustermgmt config API; host history from the clustermgmt stats API, one request per host. Host requests use a bounded worker pool and one shared request-start limiter (see Private/NutanixApi.ps1). The VM side and the cycle orchestrator live in Private/NutanixPrismCentral.ps1. Each host-stat response contains one time series per selected metric. The series are joined by timestamp so the payload builder receives one sample per bucket. #> # ---------------------------- # Phase 1: Inventory - clusters and hosts (Config APIs) # ---------------------------- function Get-NutanixClusters { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [pscustomobject]$Context ) Write-CustomLog -Message "Retrieving clusters from Prism Central" -Severity 'DEBUG' $clusters = @(Get-NutanixPaginatedResults -Context $Context -Path 'clustermgmt/v4.0/config/clusters' -ExtraQuery @{ '$select' = $script:NUTANIX_CLUSTER_SELECT }) Write-CustomLog -Message "Found $($clusters.Count) clusters" -Severity 'DEBUG' return $clusters } function Get-NutanixHosts { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [pscustomobject]$Context, [Parameter(Mandatory = $true)] [string]$ClusterExtId, [Parameter(Mandatory = $true)] [string]$ClusterName ) $path = "clustermgmt/v4.0/config/clusters/$ClusterExtId/hosts" # Request the complete host object so optional host-native values can be used when the target # PC/AOS version exposes them. No VM endpoint is needed for host-only collection. $hosts = @(Get-NutanixPaginatedResults -Context $Context -Path $path) foreach ($hostItem in $hosts) { # Annotate with cluster identity resolved from the outer loop rather than re-parsing the host's own # cluster reference - we already know it since we fetched hosts scoped to this cluster. $hostItem | Add-Member -NotePropertyName 'ClusterExtId' -NotePropertyValue $ClusterExtId -Force $hostItem | Add-Member -NotePropertyName 'ClusterName' -NotePropertyValue $ClusterName -Force } return $hosts } # ---------------------------- # Phase 2: Stats (Performance APIs) # ---------------------------- function Get-NutanixStatSamples { <# .SYNOPSIS Converts a Nutanix metric-series object into one combined sample per timestamp. .NOTES Nutanix v4 returns each selected metric as its own TimeValuePair[] property. This function joins those series by timestamp so the payload builder receives the same per-timestamp shape as the vSphere QueryPerf path. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)] [AllowNull()] [object]$StatsObject, [Parameter(Mandatory = $true)] [string[]]$FieldNames ) if ($null -eq $StatsObject) { return @() } $samplesByTimestamp = @{} foreach ($fieldName in $FieldNames) { $seriesProperty = $StatsObject.PSObject.Properties[$fieldName] foreach ($timeValuePair in @($seriesProperty.Value)) { Add-NutanixStatSample -SamplesByTimestamp $samplesByTimestamp -FieldName $fieldName -TimeValuePair $timeValuePair } } return @($samplesByTimestamp.Keys | Sort-Object | ForEach-Object { [pscustomobject]$samplesByTimestamp[$_] }) } function Add-NutanixStatSample { param( [Parameter(Mandatory = $true)] [hashtable]$SamplesByTimestamp, [Parameter(Mandatory = $true)] [string]$FieldName, [Parameter(Mandatory = $false)] [AllowNull()] [object]$TimeValuePair ) if ($null -eq $TimeValuePair -or $null -eq $TimeValuePair.timestamp) { return } try { $timestampKey = ConvertTo-NutanixTimestampKey -Value $TimeValuePair.timestamp } catch { Write-CustomLog -Message "Ignoring invalid Nutanix metric timestamp '$($TimeValuePair.timestamp)' for '$FieldName'." -Severity 'WARNING' return } if (-not $SamplesByTimestamp.ContainsKey($timestampKey)) { $SamplesByTimestamp[$timestampKey] = [ordered]@{ timestamp = $timestampKey } } $SamplesByTimestamp[$timestampKey][$FieldName] = $TimeValuePair.value } function New-NutanixHostStatsWorkItem { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [object]$Host, [Parameter(Mandatory = $true)] [datetime]$StartUtc, [Parameter(Mandatory = $true)] [datetime]$EndUtc ) $path = "clustermgmt/v4.0/stats/clusters/$($Host.ClusterExtId)/hosts/$($Host.extId)" $query = @{ '$startTime' = $StartUtc.ToString('o') '$endTime' = $EndUtc.ToString('o') '$samplingInterval' = $script:NUTANIX_STATS_SAMPLING_INTERVAL_SECONDS '$select' = $script:NUTANIX_HOST_STATS_SELECT } return [pscustomobject]@{ Host = $Host StartUtc = $StartUtc EndUtc = $EndUtc Path = $path Query = $query } } function ConvertFrom-NutanixHostStatsResponse { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [object]$Response ) $data = @($Response.data) if ($data.Count -ne 1) { throw "Expected one HostStats object but received $($data.Count)." } # The series are top-level properties of the HostStats object; tolerate a nested 'stats' wrapper too. $statsObject = $data[0] $statsProperty = $statsObject.PSObject.Properties['stats'] if ($null -ne $statsProperty -and $null -ne $statsProperty.Value -and $null -eq $statsObject.PSObject.Properties['hypervisorCpuUsagePpm']) { $statsObject = $statsProperty.Value } $fieldNames = @($script:NUTANIX_HOST_STATS_SELECT -split ',' | ForEach-Object { ($_ -split '/')[-1] }) return @(Get-NutanixStatSamples -StatsObject $statsObject -FieldNames $fieldNames) } function Get-NutanixHostStatsConcurrent { <# .SYNOPSIS Collects one host-stat request per work item with bounded concurrency and one shared request-start limiter. #> [CmdletBinding()] param( [Parameter(Mandatory = $true)] [pscustomobject]$Context, [Parameter(Mandatory = $true)] [AllowEmptyCollection()] [array]$WorkItems ) if ($WorkItems.Count -eq 0) { return [pscustomobject]@{ SamplesByHostExtId = @{}; FailedWorkItems = @() } } $baseUri = $Context.BaseUri $headers = $Context.Headers $skipCertificateCheck = [bool]$Context.SkipCertificateCheck $requestsPerSecond = [int]$script:NUTANIX_API_MAX_REQUESTS_PER_SECOND $maxRetries = [int]$script:NUTANIX_API_MAX_RETRIES $maxConcurrency = [int]$script:NUTANIX_HOST_STATS_MAX_CONCURRENCY $rateGate = $Context.RateGate $requestStarts = $Context.RequestStarts $preparedWorkItems = foreach ($workItem in $WorkItems) { $uri = "$baseUri/$($workItem.Path)" + (Build-NutanixQueryString -Query $workItem.Query) [pscustomobject]@{ Host = $workItem.Host StartUtc = $workItem.StartUtc EndUtc = $workItem.EndUtc Uri = $uri } } $rawResults = @($preparedWorkItems | ForEach-Object -ThrottleLimit $maxConcurrency -Parallel { function Wait-NutanixHostStatsRequestSlot { param($RateGate, $RequestStarts, [int]$RequestsPerSecond) do { [void]$RateGate.Wait() $waitMilliseconds = 1 $slotAcquired = $false try { $nowMilliseconds = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() [long]$oldestStart = 0 while ($RequestStarts.TryPeek([ref]$oldestStart) -and ($nowMilliseconds - $oldestStart) -ge 1000) { [long]$discardedStart = 0 $null = $RequestStarts.TryDequeue([ref]$discardedStart) } if ($RequestStarts.Count -lt $RequestsPerSecond) { $RequestStarts.Enqueue($nowMilliseconds) $slotAcquired = $true } elseif ($RequestStarts.TryPeek([ref]$oldestStart)) { $waitMilliseconds = [Math]::Max(1, 1000 - ($nowMilliseconds - $oldestStart)) } } finally { $null = $RateGate.Release() } if (-not $slotAcquired) { Start-Sleep -Milliseconds $waitMilliseconds } } while (-not $slotAcquired) } function Get-NutanixHostStatsRetryDelay { param([int]$Attempt, [Nullable[int]]$StatusCode, $Response) $backoffSeconds = [Math]::Min(30, [Math]::Pow(2, $Attempt)) if ($StatusCode -ne 429 -or $null -eq $Response.Headers) { return $backoffSeconds } $retryAfterSeconds = 0 $retryAfter = [string]$Response.Headers['Retry-After'] if ([int]::TryParse($retryAfter, [ref]$retryAfterSeconds) -and $retryAfterSeconds -gt 0) { return [Math]::Min(60, $retryAfterSeconds) } return $backoffSeconds } function New-NutanixHostStatsRequestResult { param($WorkItem, [bool]$Succeeded, [AllowNull()][string]$Content, [Nullable[int]]$StatusCode, [int]$Attempts, [AllowNull()][string]$ErrorMessage) return [pscustomobject]@{ Succeeded = $Succeeded WorkItem = $WorkItem Content = $Content StatusCode = $StatusCode Attempts = $Attempts Error = $ErrorMessage } } function Invoke-NutanixHostStatsRequest { param($WorkItem, $RateGate, $RequestStarts, [int]$RequestsPerSecond, [int]$MaxRetries, $Headers, [bool]$SkipCertificateCheck) $attempt = 0 while ($true) { $attempt++ Wait-NutanixHostStatsRequestSlot -RateGate $RateGate -RequestStarts $RequestStarts -RequestsPerSecond $RequestsPerSecond try { $requestParameters = @{ Uri = $WorkItem.Uri; Method = 'GET'; Headers = $Headers; UseBasicParsing = $true; ErrorAction = 'Stop' } if ($SkipCertificateCheck) { $requestParameters['SkipCertificateCheck'] = $true } $response = Invoke-WebRequest @requestParameters return New-NutanixHostStatsRequestResult -WorkItem $WorkItem -Succeeded $true -Content ([string]$response.Content) -StatusCode ([int]$response.StatusCode) -Attempts $attempt -ErrorMessage $null } catch { $statusCode = if ($null -ne $_.Exception.Response -and $null -ne $_.Exception.Response.StatusCode) { [int]$_.Exception.Response.StatusCode } else { $null } if ($null -ne $statusCode -and $statusCode -notin @(408, 429, 502, 503, 504)) { return New-NutanixHostStatsRequestResult -WorkItem $WorkItem -Succeeded $false -Content $null -StatusCode $statusCode -Attempts $attempt -ErrorMessage $_.Exception.Message } if ($attempt -gt $MaxRetries) { return New-NutanixHostStatsRequestResult -WorkItem $WorkItem -Succeeded $false -Content $null -StatusCode $statusCode -Attempts $attempt -ErrorMessage $_.Exception.Message } Start-Sleep -Seconds (Get-NutanixHostStatsRetryDelay -Attempt $attempt -StatusCode $statusCode -Response $_.Exception.Response) } } } Invoke-NutanixHostStatsRequest -WorkItem $_ -RateGate $using:rateGate -RequestStarts $using:requestStarts -RequestsPerSecond $using:requestsPerSecond -MaxRetries $using:maxRetries -Headers $using:headers -SkipCertificateCheck $using:skipCertificateCheck }) $samplesByHostExtId = @{} $failedWorkItems = [System.Collections.Generic.List[object]]::new() foreach ($result in $rawResults) { $hostExtId = [string]$result.WorkItem.Host.extId if (-not $result.Succeeded) { Write-CustomLog -Message "Host stats collection failed for '$hostExtId' after $($result.Attempts) attempt(s). HTTP=$($result.StatusCode) Error=$($result.Error)" -Severity 'ERROR' $failedWorkItems.Add($result.WorkItem) continue } try { $response = $result.Content | ConvertFrom-Json $samples = @(ConvertFrom-NutanixHostStatsResponse -Response $response) $samplesByHostExtId[$hostExtId] = $samples if ($result.Attempts -gt 1) { Write-CustomLog -Message "Host stats collection for '$hostExtId' succeeded after $($result.Attempts) attempts." -Severity 'WARNING' } } catch { Write-CustomLog -Message "Host stats response for '$hostExtId' could not be parsed. Error=$($_.Exception.Message)" -Severity 'ERROR' $failedWorkItems.Add($result.WorkItem) } } return [pscustomobject]@{ SamplesByHostExtId = $samplesByHostExtId FailedWorkItems = @($failedWorkItems) } } # ---------------------------- # Helpers # ---------------------------- function Get-NutanixStatValueOrNull { param( [Parameter(Mandatory = $true)] [AllowNull()] [object]$Sample, [Parameter(Mandatory = $true)] [string]$FieldName ) if ($null -eq $Sample) { return $null } $prop = $Sample.PSObject.Properties[$FieldName] if ($null -eq $prop -or $null -eq $prop.Value) { return $null } return [string]$prop.Value } function Get-NutanixSampleTimestampKey { <# .SYNOPSIS Normalizes a stat sample's 'timestamp' field into the key used to join host and VM samples. Joining is an exact value match - no nearest/fuzzy matching (see lab-validation gaps at top of file). #> param( [Parameter(Mandatory = $true)] [AllowNull()] [object]$Sample ) if ($null -eq $Sample) { return $null } $prop = $Sample.PSObject.Properties['timestamp'] if ($null -eq $prop -or $null -eq $prop.Value) { return $null } try { return ConvertTo-NutanixTimestampKey -Value $prop.Value } catch { return $null } } function ConvertTo-NutanixTimestampKey { <# .SYNOPSIS Normalizes an API timestamp into the UTC round-trip string used as a join key. .NOTES ConvertFrom-Json already turns ISO-8601 strings such as '2026-09-09T10:00:00Z' into [DateTime] values. Stringifying those drops the zone, and a zoneless string is then parsed as local time, which shifted every sample by the machine's UTC offset. DateTime values are therefore converted directly; a Kind of Unspecified is taken as UTC, which is what Prism Central emits. #> param( [Parameter(Mandatory = $true)] [AllowNull()] [object]$Value ) if ($null -eq $Value) { throw 'Timestamp is null.' } if ($Value -is [DateTimeOffset]) { return ([DateTimeOffset]$Value).UtcDateTime.ToString('o') } if ($Value -is [DateTime]) { $dateTime = [DateTime]$Value if ($dateTime.Kind -eq [DateTimeKind]::Unspecified) { $dateTime = [DateTime]::SpecifyKind($dateTime, [DateTimeKind]::Utc) } return $dateTime.ToUniversalTime().ToString('o') } return (ConvertFrom-RfcUtcTimestamp -Value ([string]$Value)).ToString('o') } function Get-HypervisorMetricsTimeRange { [CmdletBinding()] param( [Parameter(Mandatory = $true)] [datetime]$Now, [Parameter(Mandatory = $true)] [datetime]$LastReceivedUtc, [Parameter(Mandatory = $true)] [int]$MaxMinutesRead ) $start = $LastReceivedUtc # Round start to minute (floor/truncate) $start = [datetime]::new($start.Year, $start.Month, $start.Day, $start.Hour, $start.Minute, 0, [System.DateTimeKind]::Utc) # End = min(now - 1 minute, start + MaxMinutesRead) $nowMinus1 = $Now.AddMinutes(-1) $startPlusMax = $start.AddMinutes($MaxMinutesRead) $end = if ($nowMinus1 -lt $startPlusMax) { $nowMinus1 } else { $startPlusMax } if ($end -eq $startPlusMax -and $startPlusMax -lt $nowMinus1) { Write-CustomLog -Message "Metrics window capped at $MaxMinutesRead minutes: $start to $end." -Severity 'INFO' } # Round end to minute (floor/truncate) $end = [datetime]::new($end.Year, $end.Month, $end.Day, $end.Hour, $end.Minute, 0, [System.DateTimeKind]::Utc) return [pscustomobject]@{ Start = $start End = $end } } |