VBAF.RL.SettingsSearch.ps1

#Requires -Version 5.1
<#
.SYNOPSIS
    VBAF 6.2 -- Invoke-VBAFSettingsSearch: a DQN brain that finds its own settings.
.DESCRIPTION
    WHAT : tries several random settings (learning rate, gamma, epsilon decay, target update), trains a DQN brain with
            each, checks every brain on validation episodes after each round, keeps every brain's best checkpoint, and
            returns the best brain together with its settings.
    WHY : VBAF-Evolution-Lab Phase 8 showed that the default learning rate (0.001) is far too low for environments with
            small rewards (kernel finding KF-14): 0 of 12 pillar-seeds beat the best fixed action. This search lifted 12 of
            12. A population that inherits weights (population-based training) did NOT do better than this plain search.
    HOW : random search with a best checkpoint per brain. The validation seeds pick the winner -- keep your TEST seeds
            apart, or you measure your own choice instead of the brain.
    UNITS : epsilon decays once per Replay() (here every ReplayEvery steps); TargetUpdateFreq counts episodes. The learning
            rate lives in the networks, so it is set when they are created.
.EXAMPLE
    $r = Invoke-VBAFSettingsSearch -NewEnvironment { param($s) New-EnterpriseEnvironment -Name 'AlertRouter' -Seed $s }
    $r.Settings
    Measure-VBAFAgentScore -NewEnvironment { param($s) New-EnterpriseEnvironment -Name 'AlertRouter' -Seed $s } -Agent $r.Agent -Seeds (30001..30010)
#>


$script:VBAFSearchDefaultRanges = @{ LR = @(0.0005, 0.05); Gamma = @(0.8, 0.99); Decay = @(0.99, 0.9995); TUF = @(2, 20) }

function New-VBAFSearchSettings([System.Random]$Rng, $Ranges) {
    $lo = [Math]::Log([double]$Ranges.LR[0]); $hi = [Math]::Log([double]$Ranges.LR[1])
    @{ LR    = [Math]::Exp($lo + $Rng.NextDouble() * ($hi - $lo))
       Gamma = [double]$Ranges.Gamma[0] + $Rng.NextDouble() * ([double]$Ranges.Gamma[1] - [double]$Ranges.Gamma[0])
       Decay = [double]$Ranges.Decay[0] + $Rng.NextDouble() * ([double]$Ranges.Decay[1] - [double]$Ranges.Decay[0])
       TUF   = $Rng.Next([int]$Ranges.TUF[0], [int]$Ranges.TUF[1] + 1) }
}

function New-VBAFSearchBrain([int]$StateSize, [int]$ActionSize, $Settings, [int[]]$Hidden, [int]$Seed) {
    Set-VBAFSeed $Seed
    $cfg = [DQNConfig]::new(); $cfg.StateSize = $StateSize; $cfg.ActionSize = $ActionSize
    $cfg.LearningRate = [double]$Settings.LR; $cfg.Gamma = [double]$Settings.Gamma
    $cfg.EpsilonDecay = [double]$Settings.Decay; $cfg.TargetUpdateFreq = [int]$Settings.TUF
    [int[]]$arch = @($StateSize) + @($Hidden) + @($ActionSize)
    $main = [VBAFNetworkFactory]::Create($arch, $cfg.LearningRate); $tgt = [VBAFNetworkFactory]::Create($arch, $cfg.LearningRate)
    $mem = [ExperienceReplay]::new($cfg.MemorySize)
    $agent = & { [DQNAgent]::new($cfg, $main, $tgt, $mem) } 3>$null 6>$null
    $agent.SetSeed($Seed)
    return $agent
}

# Training episodes use their own seeds (1,000,000 + stream x 1000 + episode), never the validation or test seeds.
function Invoke-VBAFSearchTraining($Agent, [scriptblock]$NewEnvironment, [int]$FirstEpisode, [int]$Episodes, [int]$Stream, [int]$ReplayEvery) {
    for ($k = 0; $k -lt $Episodes; $k++) {
        $s = 1000000 + $Stream * 1000 + $FirstEpisode + $k
        Set-VBAFSeed $s
        $env = & $NewEnvironment $s
        $st = $env.Reset(); $done = $false; $steps = 0; $tot = 0.0
        while (-not $done) {
            $act = $Agent.Act($st); $r = $env.Step($act)
            $Agent.Remember($st, $act, [double]$r.Reward, $r.NextState, [bool]$r.Done)
            $steps++
            if ($steps % $ReplayEvery -eq 0) { & { [void]$Agent.Replay() } 6>$null }
            $st = $r.NextState; $done = [bool]$r.Done; $tot += [double]$r.Reward }
        & { $Agent.EndEpisode($tot) } 6>$null }
}

function Measure-VBAFAgentScore {
    param([Parameter(Mandatory)][scriptblock]$NewEnvironment, [Parameter(Mandatory)]$Agent, [Parameter(Mandatory)][int[]]$Seeds, [int]$MaxSteps = 5000)
    $t = foreach ($s in $Seeds) { Set-VBAFSeed $s; $env = & $NewEnvironment $s
        [double](Get-VBAFTrace -Environment $env -Agent $Agent -MaxSteps $MaxSteps).TotalReward }
    [Math]::Round([double](($t | Measure-Object -Average).Average), 4)
}

function Invoke-VBAFSettingsSearch {
    param(
        [Parameter(Mandatory)][scriptblock]$NewEnvironment,
        [int]$Candidates = 6,
        [int]$Episodes = 150,
        [int]$Rounds = 6,
        [int[]]$ValidationSeeds = @(20001..20020),
        [int]$Seed = 1,
        [int[]]$Hidden = @(16, 16),
        [int]$ReplayEvery = 4,
        [hashtable]$Ranges,
        [switch]$Quiet)
    if ($Candidates -lt 1) { throw 'Invoke-VBAFSettingsSearch: -Candidates must be at least 1' }
    if ($Rounds -lt 1 -or $Episodes -lt $Rounds -or ($Episodes % $Rounds) -ne 0) { throw ("Invoke-VBAFSettingsSearch: -Episodes ($Episodes) must be a multiple of -Rounds ($Rounds)") }
    if (-not $Ranges) { $Ranges = $script:VBAFSearchDefaultRanges }
    $sw = [Diagnostics.Stopwatch]::StartNew()
    Set-VBAFSeed $ValidationSeeds[0]; $probe = & $NewEnvironment $ValidationSeeds[0]
    $nS = @($probe.Reset()).Count; $nA = [int]$probe.ActionSpace.Size
    $rng = [System.Random]::new(1000 * $Seed + 17)
    $per = [int]($Episodes / $Rounds)
    $c = @(for ($i = 0; $i -lt $Candidates; $i++) {
        $set = New-VBAFSearchSettings $rng $Ranges
        [pscustomobject]@{ Index = $i; Settings = $set; Agent = (New-VBAFSearchBrain $nS $nA $set $Hidden (100 * $Seed + $i)); Best = [double]::NegativeInfinity; Snap = $null } })
    for ($r = 1; $r -le $Rounds; $r++) {
        foreach ($x in $c) {
            Invoke-VBAFSearchTraining $x.Agent $NewEnvironment (($r - 1) * $per) $per (100 * $Seed + $x.Index) $ReplayEvery
            $v = Measure-VBAFAgentScore -NewEnvironment $NewEnvironment -Agent $x.Agent -Seeds $ValidationSeeds
            if ($v -gt $x.Best) { $x.Best = $v; $x.Snap = @{ Main = $x.Agent.MainNetwork.ExportState(); Target = $x.Agent.TargetNetwork.ExportState(); Epsilon = $x.Agent.Epsilon } } }
        if (-not $Quiet) { Write-Host (' round {0}/{1}: best validation so far {2}' -f $r, $Rounds, (($c | ForEach-Object { $_.Best }) | Measure-Object -Maximum).Maximum) } }
    $w = $c | Sort-Object -Property @{ Expression = { $_.Best }; Descending = $true }, @{ Expression = { $_.Index }; Descending = $false } | Select-Object -First 1
    $w.Agent.MainNetwork.ImportState($w.Snap.Main); $w.Agent.TargetNetwork.ImportState($w.Snap.Target); $w.Agent.Epsilon = $w.Snap.Epsilon
    [pscustomobject]@{
        Agent           = $w.Agent
        Settings        = @{ LR = $w.Settings.LR; Gamma = $w.Settings.Gamma; Decay = $w.Settings.Decay; TUF = $w.Settings.TUF }
        ValidationScore = $w.Best
        Candidates      = @($c | ForEach-Object { [pscustomobject]@{ Index = $_.Index; LR = [Math]::Round($_.Settings.LR, 5); Gamma = [Math]::Round($_.Settings.Gamma, 4); Decay = [Math]::Round($_.Settings.Decay, 5); TUF = $_.Settings.TUF; BestValidation = $_.Best } })
        StateSize = $nS; ActionSize = $nA; Episodes = $Episodes; Rounds = $Rounds; ValidationSeeds = $ValidationSeeds
        Seconds = [Math]::Round($sw.Elapsed.TotalSeconds, 1) }
}