Module: PWN::AI::Agent::Policy

Defined in:
lib/pwn/ai/agent/policy.rb

Overview

PWN::AI::Agent::Policy is the LIVE tabular RL controller that pwn-ai did not have before R5. Everything else in the harness is retrieval-plus-policy: scores are written to disk and re-injected as prose, or exported later for optional LoRA. This module is the missing MDP:

state  s  — discretized (kind, task, plan, completeness, usable, last, fail)
action a  — tool name, or "final"
reward r  — step: semantic_ok hygiene; terminal: Reward.judge
next   s' — state after the tool result

Each Loop turn is one episode. Transitions land in ~/.pwn/policy_traj.jsonl. Q(s,a) and REINFORCE logits H(s,a) are updated from those tuples and persisted in ~/.pwn/policy.json.

The learned Q values are an ADVISORY term in Registry.rank. They never replace TaskSummarizer planning, plan_first, or CORE_TOOLS. Disable with PWN::Env[:agent][:policy] = false.

Constant Summary collapse

POLICY_FILE = 
File.join(Dir.home, '.pwn', 'policy.json')
TRAJECTORY_FILE = 
File.join(Dir.home, '.pwn', 'policy_traj.jsonl')
ALPHA = 
0.15
ALPHA_PG = 
0.05
GAMMA = 
0.85
EPSILON = 
0.08
STEP_OK = 
0.05
STEP_FAIL = 
-0.20
MAX_TRAJ = 
2_000
GOLD_MIN = 
0.6
VISITS_MIN = 
2
COLD_EPISODES = 
8
WARM_EPISODES = 
40
TASK_MOD = 
16
ACTION_MOD = 
16
EP_KEY = 
:pwn_policy_episode

Class Method Summary collapse

Class Method Details

.advantage(opts = {}) ⇒ Object

Q(s,a) − V(s). Unknown / cold-start pairs return 0 so rank is unchanged.



387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
 
# File 'lib/pwn/ai/agent/policy.rb', line 387

public_class_method def self.advantage(opts = {})
  return 0.0 unless enabled?

  s = opts[:state] || current_state
  a = opts[:action].to_s
  return 0.0 if s.to_s.empty? || a.empty?

  tab = load
  visits = read_visit(table: tab, state: s, action: a)
  qsa = smoothed_q(table: tab, state: s, action: a)
  # Tiny visit counts stay at 0 unless the value is already decisive.
  return 0.0 if visits < VISITS_MIN && qsa.abs < 0.08

  (qsa - max_q(table: tab, state: s)).round(4)
rescue StandardError
  0.0
end

.attach_episode!(opts = {}) ⇒ Object 



447
448
449
450
 
# File 'lib/pwn/ai/agent/policy.rb', line 447

public_class_method def self.attach_episode!(opts = {})
  Thread.current[EP_KEY] = opts[:episode]
  opts[:episode]
end

.authors ⇒ Object 



642
643
644
 
# File 'lib/pwn/ai/agent/policy.rb', line 642

public_class_method def self.authors
  "AUTHOR(S):\n  0day Inc. <support@0dayinc.com>\n"
end

.begin_episode(opts = {}) ⇒ Object

Supported Method Parameters

ep = PWN::AI::Agent::Policy.begin_episode( session_id: 'optional - PWN::Sessions id', request: 'optional - user request', kind: 'optional - request kind', intent: 'optional - Loop.request_intent', engine: 'optional - active engine', ts_state: 'optional - TaskSummarizer state hash' )



128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/pwn/ai/agent/policy.rb', line 128

public_class_method def self.begin_episode(opts = {})
  return { skipped: :disabled } unless enabled?

  sid = (opts[:session_id] || "ep_#{Thread.current.object_id}").to_s
  task = active_task_text(ts_state: opts[:ts_state], request: opts[:request])
  s0 = state(
    kind: opts[:kind],
    request: task,
    last_action: 'start',
    fails: 0,
    engine: opts[:engine],
    ts_state: opts[:ts_state]
  )
  ep = {
    session_id: sid,
    request: opts[:request].to_s[0, 240],
    kind: normalize_kind(raw: opts[:kind]),
    intent: opts[:intent].to_s,
    engine: opts[:engine].to_s,
    started_at: Time.now.utc.iso8601,
    state: s0,
    last_action: 'start',
    fails: 0,
    steps: []
  }
  Thread.current[EP_KEY] = ep
  ep
rescue StandardError => e
  warn "[pwn-ai/policy] begin_episode swallowed: #{e.class}: #{e.message}"
  nil
end

.cold? ⇒ Boolean

Returns:

  • (Boolean) 


83
84
85
86
87
 
# File 'lib/pwn/ai/agent/policy.rb', line 83

public_class_method def self.cold?
  stats[:n_episodes].to_i < COLD_EPISODES
rescue StandardError
  true
end

.current_episode ⇒ Object 



433
434
435
 
# File 'lib/pwn/ai/agent/policy.rb', line 433

public_class_method def self.current_episode
  Thread.current[EP_KEY]
end

.current_state ⇒ Object 



428
429
430
431
 
# File 'lib/pwn/ai/agent/policy.rb', line 428

public_class_method def self.current_state
  ep = Thread.current[EP_KEY]
  ep.is_a?(Hash) ? ep[:state] : nil
end

.detach_episode! ⇒ Object

Hermes split: snapshot + clear the live episode so Loop.maybe_finish_policy is a no-op on the user-visible path while TurnFinalizer re-attaches it on the background review thread.



441
442
443
444
445
 
# File 'lib/pwn/ai/agent/policy.rb', line 441

public_class_method def self.detach_episode!
  ep = Thread.current[EP_KEY]
  Thread.current[EP_KEY] = nil
  ep
end

.enabled? ⇒ Boolean

Returns:

  • (Boolean) 


629
630
631
632
633
634
635
636
637
638
639
640
 
# File 'lib/pwn/ai/agent/policy.rb', line 629

public_class_method def self.enabled?
  return true unless defined?(PWN::Env) && PWN::Env.is_a?(Hash)

  v = begin
    PWN::Env.dig(:ai, :agent, :policy)
  rescue StandardError
    nil
  end
  v.nil? || !!v
rescue StandardError
  true
end

.episode_budget_met? ⇒ Boolean

True once live returns, warmup-replayed trajectories, or a warmed Q table have enough mass to emit greedy suggestions. Cold? stays a coarser state-encoding gate; this is the banner.

Returns:

  • (Boolean) 


98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/pwn/ai/agent/policy.rb', line 98

public_class_method def self.episode_budget_met?
  n = stats[:n_episodes].to_i
  return true if n >= COLD_EPISODES

  tab = load
  warmed = !tab[:warmed_at].to_s.empty?
  pairs = 0
  tab[:q].each_value { |acts| pairs += acts.length if acts.is_a?(Hash) }
  return true if warmed && pairs >= COLD_EPISODES

  traj_n = trajectories(limit: COLD_EPISODES).length
  warmed && traj_n >= COLD_EPISODES
rescue StandardError
  false
end

.evaluate(opts = {}) ⇒ Object

Replay stored trajectories under the current Q table. Does not write. Used by task 7 (evaluate policy quality).



526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
 
# File 'lib/pwn/ai/agent/policy.rb', line 526

public_class_method def self.evaluate(opts = {})
  rows = trajectories(limit: opts[:limit] || 200)
  return { n: 0, mean_return: nil, greedy_match: nil, mean_abs_td: nil } if rows.empty?

  tab = load
  abs_td = []
  greedy_hits = 0
  greedy_n = 0
  rows.each do |ep|
    Array(ep[:steps]).each do |tr|
      s = tr[:state].to_s
      a = tr[:action].to_s
      next if s.empty? || a.empty?

      qsa = read_q(table: tab, state: s, action: a)
      max_n = tr[:terminal] ? 0.0 : max_q(table: tab, state: tr[:next_state])
      abs_td << (tr[:reward].to_f + (GAMMA * max_n) - qsa).abs
      acts = (tab[:q][s.to_sym] || {}).keys.map(&:to_s)
      next if acts.empty?

      greedy_n += 1
      best = acts.max_by { |act| read_q(table: tab, state: s, action: act) }
      greedy_hits += 1 if best == a
    end
  end
  rets = rows.map { |r| r[:return].to_f }
  {
    n: rows.length,
    mean_return: (rets.sum / rets.length).round(4),
    mean_abs_td: abs_td.empty? ? nil : (abs_td.sum / abs_td.length).round(4),
    greedy_match: greedy_n.positive? ? (greedy_hits.to_f / greedy_n).round(3) : nil
  }
rescue StandardError => e
  { n: 0, error: "#{e.class}: #{e.message}" }
end

.finish(opts = {}) ⇒ Object

Supported Method Parameters

report = PWN::AI::Agent::Policy.finish( session_id: 'optional - active episode id', score: 'optional - Reward.judge 0..1 (training target)', verdict: 'optional - solved|partial|wrong|refused', proxy_ok: 'optional - Boolean fallback when no judge score' )



236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
 
# File 'lib/pwn/ai/agent/policy.rb', line 236

public_class_method def self.finish(opts = {})
  return { skipped: :disabled } unless enabled?

  ep = Thread.current[EP_KEY]
  return { skipped: :no_episode } unless ep.is_a?(Hash)

  return { skipped: :session_mismatch } if opts[:session_id] && ep[:session_id] && opts[:session_id].to_s != ep[:session_id].to_s

  unless ep[:steps].empty?
    last = ep[:steps].last
    last[:next_state] = state(
      kind: ep[:kind],
      request: ep[:request],
      last_action: last[:action] || ep[:last_action],
      fails: ep[:fails],
      engine: ep[:engine],
      ts_state: opts[:ts_state],
      final: opts[:final],
      score: opts[:score]
    )
  end
  terminal = terminal_reward(score: opts[:score], proxy_ok: opts[:proxy_ok])
  if ep[:steps].empty?
    ep[:steps] << {
      state: ep[:state],
      action: 'final',
      reward: 0.0,
      next_state: ep[:state],
      ok: true,
      duration: 0.0,
      terminal: true
    }
  end
  ep[:steps].last[:reward] = (ep[:steps].last[:reward].to_f + terminal).round(4)
  ep[:steps].last[:terminal] = true
  ep[:score] = opts[:score]
  ep[:verdict] = opts[:verdict]
  ep[:return] = discounted_return(steps: ep[:steps])
  ep[:ended_at] = Time.now.utc.iso8601

  n_td = 0
  n_pg = 0
  ep[:steps].each_with_index do |tr, idx|
    n_td += 1 if update_q!(transition: tr)
    g = discounted_return(steps: ep[:steps][idx..])
    n_pg += 1 if update_pg!(state: tr[:state], action: tr[:action], advantage: g - value(state: tr[:state]))
  end

  persist_episode!(episode: ep)
  Thread.current[EP_KEY] = nil
  {
    session_id: ep[:session_id],
    steps: ep[:steps].length,
    return: ep[:return],
    score: opts[:score],
    td_updates: n_td,
    pg_updates: n_pg
  }
rescue StandardError => e
  warn "[pwn-ai/policy] finish swallowed: #{e.class}: #{e.message}"
  Thread.current[EP_KEY] = nil
  { error: "#{e.class}: #{e.message}" }
end

.help ⇒ Object 



646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
 
# File 'lib/pwn/ai/agent/policy.rb', line 646

public_class_method def self.help
  puts <<~USAGE
    USAGE:
      PWN::AI::Agent::Policy.begin_episode(session_id:, request:, kind:, engine:)
      PWN::AI::Agent::Policy.observe_step(action: 'shell', ok: true, duration: 0.2)
      PWN::AI::Agent::Policy.finish(session_id:, score: 0.8, verdict: :solved)
      PWN::AI::Agent::Policy.q(state:, action:)
      PWN::AI::Agent::Policy.advantage(state:, action:)   # Registry.rank term
      PWN::AI::Agent::Policy.recommend(actions: %w[shell pwn_eval])
      PWN::AI::Agent::Policy.evaluate(limit: 100)
      PWN::AI::Agent::Policy.stats
      PWN::AI::Agent::Policy.episode_budget_met?
      PWN::AI::Agent::Policy.to_context
      PWN::AI::Agent::Policy.lean!(dry_run: true)
      PWN::AI::Agent::Policy.warmup!(limit: 200)
      PWN::AI::Agent::Policy.reset

      #{self}.authors
  USAGE
end

.lean!(opts = {}) ⇒ Object 



596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
 
# File 'lib/pwn/ai/agent/policy.rb', line 596

public_class_method def self.lean!(opts = {})
  dry = opts[:dry_run] ? true : false
  return { skipped: true } unless File.exist?(TRAJECTORY_FILE)

  rows = File.readlines(TRAJECTORY_FILE)
  keep = []
  rows.each do |line|
    ep = JSON.parse(line, symbolize_names: true)
    gold = ep[:return].to_f >= GOLD_MIN || ep[:score].to_f >= GOLD_MIN
    keep << [ep, line, gold]
  rescue StandardError
    next
  end
  gold = keep.select { |_, _, g| g }.map { |_, line, _| line }
  rest = keep.reject { |_, _, g| g }.last([MAX_TRAJ - gold.length, 0].max).map { |_, line, _| line }
  out = gold + rest
  unless dry
    tmp = "#{TRAJECTORY_FILE}.#{Process.pid}.tmp"
    File.write(tmp, out.join)
    File.rename(tmp, TRAJECTORY_FILE)
  end
  { removed: rows.length - out.length, remaining: out.length, dry_run: dry }
rescue StandardError => e
  { error: "#{e.class}: #{e.message}" }
end

.load ⇒ Object

Persistence / eval



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'lib/pwn/ai/agent/policy.rb', line 456

public_class_method def self.load
  FileUtils.mkdir_p(File.dirname(POLICY_FILE))
  return blank_table unless File.exist?(POLICY_FILE)

  data = JSON.parse(File.read(POLICY_FILE), symbolize_names: true)
  data[:q] = {} unless data[:q].is_a?(Hash)
  data[:h] = {} unless data[:h].is_a?(Hash)
  data[:visits] = {} unless data[:visits].is_a?(Hash)
  data[:returns] = Array(data[:returns])
  data
rescue StandardError
  blank_table
end

.maybe_warmup! ⇒ Object 



917
918
919
920
921
922
923
924
925
926
927
928
929
 
# File 'lib/pwn/ai/agent/policy.rb', line 917

public_class_method def self.maybe_warmup!
  return { skipped: :disabled } unless enabled?
  return { skipped: :no_traj } unless File.exist?(TRAJECTORY_FILE)

  tab = load
  pairs = 0
  tab[:q].each_value { |acts| pairs += acts.length if acts.is_a?(Hash) }
  return { skipped: :warmed } if !tab[:warmed_at].to_s.empty? && pairs >= COLD_EPISODES && !cold?

  warmup!
rescue StandardError
  { skipped: :error }
end

.observe_step(opts = {}) ⇒ Object

Supported Method Parameters

step = PWN::AI::Agent::Policy.observe_step( session_id: 'optional - must match begin_episode when set', action: 'required - tool name', ok: 'required - Boolean, Reward.semantic_ok', duration: 'optional - Float seconds', ts_state: 'optional - TaskSummarizer state', request: 'optional - used if episode was not begun', kind: 'optional', engine: 'optional' )



172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
 
# File 'lib/pwn/ai/agent/policy.rb', line 172

public_class_method def self.observe_step(opts = {})
  return { skipped: :disabled } unless enabled?

  action = opts[:action].to_s
  return { skipped: :no_action } if action.empty?

  ep = Thread.current[EP_KEY]
  if ep.nil? || (opts[:session_id] && ep[:session_id] && opts[:session_id].to_s != ep[:session_id].to_s)
    begin_episode(
      session_id: opts[:session_id],
      request: opts[:request],
      kind: opts[:kind],
      engine: opts[:engine],
      ts_state: opts[:ts_state]
    )
    ep = Thread.current[EP_KEY]
  end
  return { skipped: :no_episode } unless ep.is_a?(Hash)

  ok = opts[:ok] ? true : false
  ep[:fails] = ep[:fails].to_i + 1 unless ok
  distrust = 0.0
  distrust = Reward.proxy_distrust.to_f if defined?(Reward) && Reward.respond_to?(:proxy_distrust)
  reward = if distrust >= 0.85
             0.0
           else
             ok ? STEP_OK : STEP_FAIL
           end
  s = ep[:state]
  task = active_task_text(ts_state: opts[:ts_state], request: ep[:request])
  s2 = state(
    kind: ep[:kind],
    request: task,
    last_action: action,
    fails: ep[:fails],
    engine: ep[:engine],
    ts_state: opts[:ts_state]
  )
  trans = {
    state: s,
    action: action,
    reward: reward,
    next_state: s2,
    ok: ok,
    duration: opts[:duration].to_f,
    terminal: false
  }
  ep[:steps] << trans
  ep[:state] = s2
  ep[:last_action] = action
  trans
rescue StandardError => e
  warn "[pwn-ai/policy] observe_step swallowed: #{e.class}: #{e.message}"
  nil
end

.q(opts = {}) ⇒ Object

Query — used by Registry.rank (advisory only)



373
374
375
376
377
 
# File 'lib/pwn/ai/agent/policy.rb', line 373

public_class_method def self.q(opts = {})
  read_q(table: load, state: opts[:state], action: opts[:action])
rescue StandardError
  0.0
end

.recommend(opts = {}) ⇒ Object

Supported Method Parameters

pick = PWN::AI::Agent::Policy.recommend( state: 'optional - default current episode state', actions: 'required - Array of tool names', epsilon: 'optional - explore probability (default EPSILON)' )



412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
 
# File 'lib/pwn/ai/agent/policy.rb', line 412

public_class_method def self.recommend(opts = {})
  actions = Array(opts[:actions]).map(&:to_s).reject(&:empty?)
  return { action: nil, reason: :empty } if actions.empty?

  s = opts[:state] || current_state || 'unknown'
  eps = opts.key?(:epsilon) ? opts[:epsilon].to_f : EPSILON
  return { action: actions.sample, reason: :explore, state: s, epsilon: eps } if rand < eps

  tab = load
  scored = actions.map { |a| [a, smoothed_q(table: tab, state: s, action: a)] }
  best = scored.max_by { |_, v| v }
  { action: best[0], q: best[1].round(4), reason: :greedy, state: s, ranked: scored.sort_by { |_, v| -v } }
rescue StandardError => e
  { action: Array(opts[:actions]).first, reason: :error, error: e.message }
end

.reset ⇒ Object 



622
623
624
625
626
627
 
# File 'lib/pwn/ai/agent/policy.rb', line 622

public_class_method def self.reset
  FileUtils.rm_f(POLICY_FILE)
  FileUtils.rm_f(TRAJECTORY_FILE)
  Thread.current[EP_KEY] = nil
  blank_table
end

.save(opts = {}) ⇒ Object 



470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
 
# File 'lib/pwn/ai/agent/policy.rb', line 470

public_class_method def self.save(opts = {})
  table = opts[:table] || load
  table[:updated_at] = Time.now.utc.iso8601
  FileUtils.mkdir_p(File.dirname(POLICY_FILE))
  path = POLICY_FILE
  tmp = File.join(File.dirname(path), ".#{File.basename(path)}.#{Process.pid}.tmp")
  File.open(tmp, File::WRONLY | File::CREAT | File::TRUNC, 0o644) do |f|
    f.flock(File::LOCK_EX)
    f.write(JSON.pretty_generate(table))
    f.flush
    f.fsync
  end
  File.rename(tmp, path)
  table
ensure
  FileUtils.rm_f(tmp) if defined?(tmp) && tmp && File.exist?(tmp)
end

.state(opts = {}) ⇒ Object

Supported Method Parameters

key = PWN::AI::Agent::Policy.state( kind: 'optional - statement|question|autonomous_goal|…', request: 'optional - user text / active English task', last_action: 'optional - previous tool name', fails: 'optional - in-turn failure count', engine: 'optional - active engine' )



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/pwn/ai/agent/policy.rb', line 61

public_class_method def self.state(opts = {})
  kind = normalize_kind(raw: opts[:kind] || opts[:request_kind])
  task = task_family(text: opts[:task] || opts[:request])
  eng  = opts[:engine].to_s.empty? ? 'any' : opts[:engine].to_s.downcase
  plan_q = plan_quality_bin(ts_state: opts[:ts_state])
  comp = completeness_bin(final: opts[:final], score: opts[:score])
  use = usable_bin(final: opts[:final], score: opts[:score])
  # Three independent bins so Q can see plan quality, answer
  # completeness, and whether the human actually got a usable result.
  qual = "p#{plan_q}c#{comp}u#{use}"
  if warm?
    last = action_bucket(name: opts[:last_action] || opts[:last] || 'start')
    fail = fail_bin(count: opts[:fails] || opts[:fail_n])
    "#{kind}|#{task}|a#{last}|f#{fail}|#{qual}|#{eng}"
  elsif !cold?
    fail = fail_bin(count: opts[:fails] || opts[:fail_n])
    "#{kind}|#{task}|f#{fail}|#{qual}|#{eng}"
  else
    "#{kind}|#{task}|#{qual}|#{eng}"
  end
end

.stats ⇒ Object 



501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
 
# File 'lib/pwn/ai/agent/policy.rb', line 501

public_class_method def self.stats
  tab = load
  q_pairs = 0
  tab[:q].each_value { |acts| q_pairs += acts.length if acts.is_a?(Hash) }
  rets = Array(tab[:returns])
  mean_r = rets.empty? ? nil : (rets.sum.to_f / rets.length).round(4)
  n_up = tab[:n_updates].to_i
  td_mean = n_up.positive? ? (tab[:td_abs_sum].to_f / n_up).round(4) : nil
  {
    enabled: enabled?,
    n_updates: n_up,
    n_states: tab[:q].length,
    n_pairs: q_pairs,
    n_episodes: rets.length,
    mean_return: mean_r,
    mean_abs_td: td_mean,
    alpha: ALPHA,
    gamma: GAMMA,
    epsilon: EPSILON
  }
end

.to_context(opts = {}) ⇒ Object 



562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
 
# File 'lib/pwn/ai/agent/policy.rb', line 562

public_class_method def self.to_context(opts = {})
  return '' unless enabled?

  maybe_warmup! unless episode_budget_met?
  s = stats
  lines = []
  unless episode_budget_met?
    lines << 'POLICY (R5 tabular Q / REINFORCE — advisory only, does not replace planning)'
    lines << "  policy cold episodes=#{s[:n_episodes]}/#{COLD_EPISODES} — omit greedy suggestion"
    return "#{lines.join("\n")}\n"
  end

  ev = evaluate(limit: opts[:limit] || 40)
  fallback = %w[memory_recall sessions_view pwn_eval shell mistakes_record mistakes_resolve learning_note_outcome memory_remember]
  pref = begin
    PWN::AI::Agent::Registry.preference_order
  rescue StandardError
    []
  end
  actions = pref.empty? ? fallback : pref
  rec = begin
    recommend(actions: actions, epsilon: 0.0)[:action]
  rescue StandardError
    nil
  end
  lines << 'POLICY (R5 tabular Q / REINFORCE — advisory only, does not replace planning)'
  lines << "  episodes=#{s[:n_episodes]} states=#{s[:n_states]} pairs=#{s[:n_pairs]} updates=#{s[:n_updates]}"
  lines << "  mean_return=#{s[:mean_return] || '-'} mean|TD|=#{s[:mean_abs_td] || '-'} greedy_match=#{ev[:greedy_match] || '-'}"
  lines << "  current_state=#{current_state || '(none)'} suggest=#{rec || '-'} tool_preference=#{actions.join(',')}"
  "#{lines.join("\n")}\n"
rescue StandardError
  ''
end

.trajectories(opts = {}) ⇒ Object 



488
489
490
491
492
493
494
495
496
497
498
499
 
# File 'lib/pwn/ai/agent/policy.rb', line 488

public_class_method def self.trajectories(opts = {})
  limit = (opts[:limit] || 50).to_i
  return [] unless File.exist?(TRAJECTORY_FILE)

  File.readlines(TRAJECTORY_FILE).last(limit).filter_map do |line|
    JSON.parse(line, symbolize_names: true)
  rescue StandardError
    nil
  end.reverse
rescue StandardError
  []
end

.update_pg!(opts = {}) ⇒ Object

Supported Method Parameters

h = PWN::AI::Agent::Policy.update_pg!( state: 'required', action: 'required', advantage: 'required - scalar G_t − V(s)' )



344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
 
# File 'lib/pwn/ai/agent/policy.rb', line 344

public_class_method def self.update_pg!(opts = {})
  return nil unless enabled?

  s = opts[:state].to_s
  a = opts[:action].to_s
  adv = opts[:advantage].to_f
  return nil if s.empty? || a.empty?
  return 0.0 if adv.abs < 1e-9

  tab = load
  logits = (tab[:h][s.to_sym] || {}).dup
  logits[a.to_sym] = logits[a.to_sym].to_f
  pi = softmax(logits: logits)
  logits.each_key do |act|
    grad = act.to_s == a ? (1.0 - pi[act].to_f) : -pi[act].to_f
    logits[act] = logits[act].to_f + (ALPHA_PG * adv * grad)
  end
  tab[:h][s.to_sym] = logits
  save(table: tab)
  logits[a.to_sym].to_f.round(5)
rescue StandardError => e
  warn "[pwn-ai/policy] update_pg! swallowed: #{e.class}: #{e.message}"
  nil
end

.update_q!(opts = {}) ⇒ Object

Supported Method Parameters

q = PWN::AI::Agent::Policy.update_q!( transition: 'required - Hash with :state :action :reward :next_state :terminal' )



309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
 
# File 'lib/pwn/ai/agent/policy.rb', line 309

public_class_method def self.update_q!(opts = {})
  return nil unless enabled?

  tr = opts[:transition] || opts
  s  = tr[:state].to_s
  a  = tr[:action].to_s
  return nil if s.empty? || a.empty?

  r  = tr[:reward].to_f
  s2 = tr[:next_state].to_s
  term = tr[:terminal] ? true : false
  tab = load
  qsa = read_q(table: tab, state: s, action: a)
  max_n = term ? 0.0 : max_q(table: tab, state: s2)
  target = r + (GAMMA * max_n)
  td = target - qsa
  new_q = qsa + (ALPHA * td)
  write_q!(table: tab, state: s, action: a, value: new_q)
  bump_visit!(table: tab, state: s, action: a)
  tab[:n_updates] = tab[:n_updates].to_i + 1
  tab[:td_abs_sum] = tab[:td_abs_sum].to_f + td.abs
  save(table: tab)
  new_q.round(5)
rescue StandardError => e
  warn "[pwn-ai/policy] update_q! swallowed: #{e.class}: #{e.message}"
  nil
end

.value(opts = {}) ⇒ Object 



379
380
381
382
383
 
# File 'lib/pwn/ai/agent/policy.rb', line 379

public_class_method def self.value(opts = {})
  max_q(table: load, state: opts[:state])
rescue StandardError
  0.0
end

.warm? ⇒ Boolean

Returns:

  • (Boolean) 


89
90
91
92
93
 
# File 'lib/pwn/ai/agent/policy.rb', line 89

public_class_method def self.warm?
  stats[:n_episodes].to_i >= WARM_EPISODES
rescue StandardError
  false
end

.warmup!(opts = {}) ⇒ Object

Replay stored trajectories into Q so a cold table is not empty advice.



872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
 
# File 'lib/pwn/ai/agent/policy.rb', line 872

public_class_method def self.warmup!(opts = {})
  return { skipped: :disabled } unless enabled?
  return { skipped: :no_traj } unless File.exist?(TRAJECTORY_FILE)

  tab = load
  rows = trajectories(limit: opts[:limit] || 400)
  n = 0
  2.times do
    rows.reverse_each do |ep|
      Array(ep[:steps]).each do |tr|
        s = tr[:state].to_s
        a = tr[:action].to_s
        next if s.empty? || a.empty?

        r = tr[:reward].to_f
        s2 = tr[:next_state].to_s
        term = tr[:terminal] ? true : false
        qsa = read_q(table: tab, state: s, action: a)
        max_n = term ? 0.0 : max_q(table: tab, state: s2)
        target = r + (GAMMA * max_n)
        td = target - qsa
        write_q!(table: tab, state: s, action: a, value: qsa + (ALPHA * td))
        bump_visit!(table: tab, state: s, action: a)
        tab[:n_updates] = tab[:n_updates].to_i + 1
        tab[:td_abs_sum] = tab[:td_abs_sum].to_f + td.abs
        n += 1
      end
    end
  end
  # Credit stored returns toward the episode budget so greedy
  # suggestions are not omitted after a successful replay of a
  # table that never finished COLD_EPISODES live turns.
  rets = Array(tab[:returns])
  need = COLD_EPISODES - rets.length
  if need.positive?
    extras = rows.filter_map { |ep| ep[:return] unless ep[:return].nil? }.first(need)
    tab[:returns] = (rets + extras).last(200)
  end
  tab[:warmed_at] = Time.now.utc.iso8601
  save(table: tab)
  { replayed: rows.length, td_updates: n, n_episodes: Array(load[:returns]).length }
rescue StandardError => e
  { error: "#{e.class}: #{e.message}" }
end