VBAF.RL.Trace.ps1
|
#Requires -Version 5.1 <# .SYNOPSIS Get-VBAFTrace -- run an agent or a policy through one episode of ANY VBAF environment and record every step. .DESCRIPTION VBAF environments follow one of two contracts (found while building v6.0): style A Step($a) returns @{ NextState; Reward; Done } (VBAFEnvironment and its children, DQN/PPO/A3C) style B Step($a) returns nothing; read LastReward, LastDone (the enterprise pillar environments) and GetState() afterwards The style is detected after the first step and reported. An environment without Step/Reset (style C, e.g. the Market and CastleCompetition simulations) gives a clear error instead of an empty trace. -Snapshot is an optional scriptblock param($env) taken BEFORE every step and once at the end, so an environment can add its own details (the value "after" step k is snapshot k+1). .EXAMPLE $env = New-VBAFEnvironment -Name GridWorld -MaxSteps 50 $t = Get-VBAFTrace -Environment $env -Agent $agent $t.Steps | Format-Table Step, Action, Reward, Total, Done #> function Get-VBAFTrace { param( [Parameter(Mandatory = $true)] $Environment, $Agent = $null, [scriptblock] $Policy = $null, [int] $Seed = -1, [int] $MaxSteps = 10000, [scriptblock] $Snapshot = $null, [int] $PolicySeed = 7 ) if ($null -eq $Agent -and $null -eq $Policy) { throw 'Get-VBAFTrace: give -Agent or -Policy.' } $e = $Environment $methods = @($e.PSObject.Methods.Name) if (-not ($methods -contains 'Step') -or -not ($methods -contains 'Reset')) { throw ('Get-VBAFTrace: {0} has no Step/Reset (style C) and cannot be traced.' -f $e.GetType().Name) } $seedMethod = 'none' if ($Seed -ge 0 -and ($methods -contains 'ResetWithSeed')) { $state = $e.ResetWithSeed($Seed); $seedMethod = 'ResetWithSeed' } else { if ($Seed -ge 0) { if (Get-Command Set-VBAFSeed -ErrorAction SilentlyContinue) { Set-VBAFSeed $Seed; $seedMethod = 'Set-VBAFSeed' } else { Get-Random -SetSeed $Seed | Out-Null; $seedMethod = 'Get-Random' } } $state = $e.Reset() } $rng = [System.Random]::new($PolicySeed) $steps = [System.Collections.Generic.List[object]]::new() $snaps = [System.Collections.Generic.List[object]]::new() $total = 0.0; $done = $false; $style = ''; $k = 0 while (-not $done -and $k -lt $MaxSteps) { if ($null -ne $Snapshot) { $snaps.Add((& $Snapshot $e)) } $s0 = [double[]]@($state) if ($null -ne $Agent) { $a = [int]$Agent.Predict($s0) } else { $a = [int](& $Policy $s0 $rng) } $ret = $e.Step($a) if ($ret -is [hashtable] -and $ret.ContainsKey('NextState') -and $ret.ContainsKey('Reward') -and $ret.ContainsKey('Done')) { $thisStyle = 'A'; $r = [double]$ret.Reward; $done = [bool]$ret.Done; $state = $ret.NextState } elseif ((@($e.PSObject.Properties.Name) -contains 'LastReward') -and (@($e.PSObject.Properties.Name) -contains 'LastDone')) { $thisStyle = 'B'; $r = [double]$e.LastReward; $done = [bool]$e.LastDone; $state = $e.GetState() } else { throw ('Get-VBAFTrace: {0}.Step() returned neither NextState/Reward/Done (style A) nor set LastReward/LastDone (style B).' -f $e.GetType().Name) } if ($style -eq '') { $style = $thisStyle } $total += $r; $k++ $steps.Add([pscustomobject]@{ Step = $k; State = $s0; Action = $a; Reward = $r; Total = $total; Done = $done }) } if ($null -ne $Snapshot) { $snaps.Add((& $Snapshot $e)) } return [pscustomobject]@{ Environment = $e.GetType().Name; Style = $style; Seed = $Seed; SeedMethod = $seedMethod Steps = @($steps); StepCount = $k; TotalReward = $total; Finished = $done; Snapshots = @($snaps) } } |