VBAF.RL.ProductionCell.ps1
|
#Requires -Version 5.1 <# .SYNOPSIS The production cell: a small, honest world for learning to schedule (VBAF v6.0). .DESCRIPTION One machine, an order queue, deadlines. A shift is one episode of 200 minutes; orders arrive at random, each with a processing time and a deadline. The brain sees the first 3 orders in the queue and picks one. On time: +1. Late: a penalty that grows with the lateness. The same seed gives exactly the same shift. Test shifts: seeds 1001-1030 (never for training or selection). Validation shifts: seeds 2001-2010. Hand-written rules for comparison: Random, FIFO, EDF, SPT (shortest job first, "Brain 0"), LST. Comes from VBAF-Evolution-Lab (phases 1-5), where it was used to prove that the kernel can build a brain, measure it and evolve a better one. .EXAMPLE $world = [ProductionCellEnvironment]::new(1) Measure-VBAFProductionPolicy -World $world -Policy (Get-VBAFProductionRules)['SPT'].Policy -Seeds (Get-VBAFProductionTestSeeds) #> class ProductionCellEnvironment : VBAFEnvironment { # --- World settings --- [int] $ShiftLength = 200 # ticks per shift (one episode) [double] $ArrivalProb = 0.30 # chance of a new order each tick [int] $MinProc = 1 [int] $MaxProc = 5 [int] $MinSlack = 2 [int] $MaxSlack = 15 [int] $Slots = 3 # orders the brain can see and choose from # --- Live state --- [int] $Clock [int] $Seed hidden [System.Random] $Rng hidden [System.Collections.Generic.List[hashtable]] $Pending # generated, not yet arrived hidden [System.Collections.Generic.List[hashtable]] $Queue # arrived, waiting # --- Shift statistics --- [int] $Arrived [int] $OnTime [int] $Late [int] $InvalidChoices [int] $IdleTicks [double] $TotalLateness ProductionCellEnvironment([int]$seed) : base("ProductionCell", 1000) { $this.Seed = $seed $this.Rng = [System.Random]::new($seed) $this.ObservationSpace = [VBAFSpace]::new("continuous", 11, -1.0, 1.0) $this.ActionSpace = [VBAFSpace]::new("discrete", 3, 0.0, 1.0) $this.Reset() } # Next shift, continuing the random stream (variety during training) [double[]] Reset() { $this.Clock = 0 $this.Steps = 0 $this.TotalReward = 0.0 $this.EpisodeCount++ $this.Arrived = 0 $this.OnTime = 0 $this.Late = 0 $this.InvalidChoices = 0 $this.IdleTicks = 0 $this.TotalLateness = 0.0 $this.Pending = [System.Collections.Generic.List[hashtable]]::new() $this.Queue = [System.Collections.Generic.List[hashtable]]::new() $id = 0 for ($t = 0; $t -lt $this.ShiftLength; $t++) { if ($this.Rng.NextDouble() -lt $this.ArrivalProb) { $proc = $this.Rng.Next($this.MinProc, $this.MaxProc + 1) $slack = $this.Rng.Next($this.MinSlack, $this.MaxSlack + 1) $id++ $this.Pending.Add(@{ Id = $id; Arrival = $t; Proc = $proc; Deadline = $t + $proc + $slack }) } } $this.AdmitArrivals() return $this.GetState() } # Fixed shift: the same seed always gives the exact same orders (testing and measuring) [double[]] ResetWithSeed([int]$seed) { $this.Seed = $seed $this.Rng = [System.Random]::new($seed) return $this.Reset() } hidden [void] AdmitArrivals() { while ($this.Pending.Count -gt 0 -and $this.Pending[0].Arrival -le $this.Clock) { $this.Queue.Add($this.Pending[0]) $this.Pending.RemoveAt(0) $this.Arrived++ } } [double[]] GetState() { $s = [double[]]::new(11) for ($i = 0; $i -lt $this.Slots; $i++) { if ($i -lt $this.Queue.Count) { $o = $this.Queue[$i] $slack = [int]$o.Deadline - $this.Clock - [int]$o.Proc $s[$i * 3] = 1.0 $s[$i * 3 + 1] = [double]$o.Proc / $this.MaxProc $s[$i * 3 + 2] = [Math]::Max(-1.0, [Math]::Min(1.0, [double]$slack / $this.MaxSlack)) } } $s[9] = [Math]::Min(1.0, [double]$this.Queue.Count / 10.0) $s[10] = [Math]::Max(0.0, [double]($this.ShiftLength - $this.Clock) / $this.ShiftLength) return $s } [hashtable] Step([int]$action) { $this.Steps++ $reward = 0.0 $choosable = [Math]::Min($this.Slots, $this.Queue.Count) if ($this.Queue.Count -eq 0) { # Nothing to do: machine waits one tick $this.Clock++ $this.IdleTicks++ } elseif ($action -lt 0 -or $action -ge $choosable) { # Picked an empty slot: wasted tick $reward = -0.2 $this.Clock++ $this.IdleTicks++ $this.InvalidChoices++ } else { $o = $this.Queue[$action] $this.Queue.RemoveAt($action) $this.Clock += [int]$o.Proc $lateness = $this.Clock - [int]$o.Deadline if ($lateness -le 0) { $this.OnTime++ $reward = 1.0 } else { $this.Late++ $this.TotalLateness += $lateness $reward = -(0.5 + [Math]::Min([int]$lateness, 10) / 20.0) } } $this.AdmitArrivals() $done = ($this.Clock -ge $this.ShiftLength) -or ($this.Steps -ge $this.MaxSteps) if ($done) { # Orders still waiting and already past their deadline count against the brain foreach ($q in $this.Queue) { if ([int]$q.Deadline -lt $this.Clock) { $reward -= 0.5 } } } $this.TotalReward += $reward return @{ NextState = $this.GetState(); Reward = $reward; Done = $done } } [hashtable] GetShiftStats() { $total = $this.Arrived + $this.Pending.Count $pct = 0.0 if ($total -gt 0) { $pct = [Math]::Round(100.0 * $this.OnTime / $total, 1) } $avgLate = 0.0 if ($this.Late -gt 0) { $avgLate = [Math]::Round($this.TotalLateness / $this.Late, 2) } return @{ Orders = $total OnTime = $this.OnTime Late = $this.Late Unfinished = $this.Queue.Count + $this.Pending.Count OnTimePct = $pct AvgLateness = $avgLate Invalid = $this.InvalidChoices IdleTicks = $this.IdleTicks Clock = $this.Clock TotalReward = [Math]::Round($this.TotalReward, 2) } } } function Get-VBAFProductionTestSeeds { return [int[]](1001..1030) } # Pick the visible slot (0-2) with the LOWEST key value. Ties go to the oldest order. function Select-VBAFProductionSlot { param([double[]]$State, [scriptblock]$Key) $best = 0 $bestVal = [double]::MaxValue for ($i = 0; $i -lt 3; $i++) { if ($State[$i * 3] -gt 0.5) { $v = & $Key $State[$i * 3 + 1] $State[$i * 3 + 2] if ($v -lt $bestVal) { $bestVal = $v; $best = $i } } } return $best } # Hand-written rules. Each policy: param($s, $rng) -> action (0-2). # State per slot: present, proc/5, slack/15 (time to deadline = proc + slack) function Get-VBAFProductionRules { $rules = [ordered]@{} $rules['Random'] = @{ Description = 'Random choice (floor)' Policy = { param($s, $rng) $valid = [int]$s[0] + [int]$s[3] + [int]$s[6] if ($valid -eq 0) { 0 } else { $rng.Next(0, $valid) } } } $rules['FIFO'] = @{ Description = 'Oldest order first' Policy = { param($s, $rng) 0 } } $rules['EDF'] = @{ Description = 'Earliest deadline first' Policy = { param($s, $rng) Select-VBAFProductionSlot -State $s -Key { param($p, $sl) $p * 5 + $sl * 15 } } } $rules['SPT'] = @{ Description = 'Shortest job first' Policy = { param($s, $rng) Select-VBAFProductionSlot -State $s -Key { param($p, $sl) $p } } } $rules['LST'] = @{ Description = 'Least slack first' Policy = { param($s, $rng) Select-VBAFProductionSlot -State $s -Key { param($p, $sl) $sl } } } return $rules } # Run a policy over every test seed and summarise. function Measure-VBAFProductionPolicy { param($World, [scriptblock]$Policy, [int[]]$Seeds, [int]$PolicySeed = 7) $rng = [System.Random]::new($PolicySeed) $rows = foreach ($seed in $Seeds) { $state = $World.ResetWithSeed($seed) $done = $false while (-not $done) { $a = [int](& $Policy $state $rng) $r = $World.Step($a) $state = $r.NextState $done = $r.Done } $st = $World.GetShiftStats() [pscustomobject]@{ Seed = $seed; Reward = $st.TotalReward; OnTimePct = $st.OnTimePct; AvgLateness = $st.AvgLateness } } $rw = @($rows | ForEach-Object { [double]$_.Reward }) $mean = ($rw | Measure-Object -Average).Average $var = ($rw | ForEach-Object { ($_ - $mean) * ($_ - $mean) } | Measure-Object -Average).Average return [pscustomobject]@{ Score = [Math]::Round($mean, 2) ScoreStd = [Math]::Round([Math]::Sqrt($var), 2) MinReward = ($rw | Measure-Object -Minimum).Minimum MaxReward = ($rw | Measure-Object -Maximum).Maximum OnTimePct = [Math]::Round(($rows | Measure-Object -Property OnTimePct -Average).Average, 1) AvgLateness = [Math]::Round(($rows | Measure-Object -Property AvgLateness -Average).Average, 2) Seeds = $Seeds.Count PerSeed = $rows } } # Validation shifts (seeds 2001-2010): for checkpoints and fitness, never the test set. function Get-VBAFProductionValidationSeeds { return [int[]](2001..2010) } # ---------- v6.0: a production-cell shift, step by step (for the window's "The shift" and "Side by side") ---------- # Built on the general Get-VBAFTrace plus a production-cell snapshot (clock, queue length, the first Slots orders). # Each step gets: ClockBefore/After, QueueLen, Visible (Id, Proc, Deadline, Slack), Action, ChosenId, # Outcome (ontime | late | idle = empty queue, not the brain's fault | invalid = an empty slot was picked), Reward, Total. function Get-VBAFShiftTrace { param($World, $Agent = $null, [scriptblock]$Policy = $null, [int]$Seed, [int]$PolicySeed = 7) $snap = { param($e) $q = $e.Queue; $n = [Math]::Min([int]$e.Slots, [int]$q.Count) $vis = @(for ($i = 0; $i -lt $n; $i++) { $o = $q[$i]; [pscustomobject]@{ Id = [int]$o.Id; Proc = [int]$o.Proc; Deadline = [int]$o.Deadline; Slack = ([int]$o.Deadline - [int]$e.Clock - [int]$o.Proc) } }) [pscustomobject]@{ Clock = [int]$e.Clock; QueueLen = [int]$q.Count; Visible = $vis } } if ($null -ne $Agent) { $tr = Get-VBAFTrace -Environment $World -Agent $Agent -Seed $Seed -Snapshot $snap -PolicySeed $PolicySeed } else { $tr = Get-VBAFTrace -Environment $World -Policy $Policy -Seed $Seed -Snapshot $snap -PolicySeed $PolicySeed } $sn = @($tr.Snapshots) $steps = [System.Collections.Generic.List[object]]::new() $st = @($tr.Steps) for ($k = 0; $k -lt $st.Count; $k++) { $s = $st[$k]; $b = $sn[$k]; $a = $sn[$k + 1] $vis = @($b.Visible) $chosenId = 0; $chosenDeadline = 0 if ($b.QueueLen -gt 0 -and $s.Action -ge 0 -and $s.Action -lt $vis.Count) { $chosenId = $vis[$s.Action].Id; $chosenDeadline = $vis[$s.Action].Deadline } $outcome = 'idle' if ($b.QueueLen -gt 0) { if ($chosenId -eq 0) { $outcome = 'invalid' } elseif ([int]$a.Clock -le $chosenDeadline) { $outcome = 'ontime' } else { $outcome = 'late' } } $steps.Add([pscustomobject]@{ Step = $k + 1; ClockBefore = [int]$b.Clock; ClockAfter = [int]$a.Clock; QueueLen = [int]$b.QueueLen Visible = $vis; Action = [int]$s.Action; ChosenId = $chosenId; Outcome = $outcome; Reward = [double]$s.Reward; Total = [double]$s.Total }) } return [pscustomobject]@{ Seed = $Seed; Steps = @($steps); Stats = $World.GetShiftStats() } } # The brains of a study (ResultDir of Invoke-VBAFEvolutionStudy): Brain 0 (the SPT rule), and -- when the final-test # models exist -- the control (baseline genome) and the champion, both for the FIRST final seed. function Get-VBAFShiftBrains([string]$ResultDir) { $b = [ordered]@{} $b['Brain 0 (SPT rule)'] = @{ Policy = (Get-VBAFProductionRules)['SPT'].Policy } $sp = $null foreach ($n in 'evolution-summary.json', 'phase4c.json') { $p = Join-Path $ResultDir $n; if (Test-Path $p) { $sp = $p; break } } if ($null -eq $sp) { return $b } $s = (Get-Content $sp -Raw -Encoding UTF8 | ConvertFrom-Json) $seed = [int]@($s.FinalSeeds)[0] $cm = Join-Path $ResultDir ('final-control-s{0}-best.xml' -f $seed) if (Test-Path $cm) { $b[('Control (baseline genome, seed {0})' -f $seed)] = @{ Agent = (Restore-VBAFBrain -Genome (Get-VBAFBaselineGenome) -BrainSeed $seed -ModelPath $cm) } } $hm = Join-Path $ResultDir ('final-champion-s{0}-best.xml' -f $seed) if (Test-Path $hm) { $b[('Champion {0} (seed {1})' -f $s.Champion, $seed)] = @{ Agent = (Restore-VBAFBrain -Genome (ConvertTo-VBAFGenome $s.ChampionGenome) -BrainSeed $seed -ModelPath $hm) } } return $b } function Get-VBAFShiftTraceFor($World, $Brain, [int]$Seed) { if ($Brain.ContainsKey('Agent')) { return Get-VBAFShiftTrace -World $World -Agent $Brain.Agent -Seed $Seed } return Get-VBAFShiftTrace -World $World -Policy $Brain.Policy -Seed $Seed } |