Module: PWN::AI::Agent::Policy

Defined in:
lib/pwn/ai/agent/policy.rb

Overview

PWN::AI::Agent::Policy is the LIVE tabular RL controller that pwn-ai did not have before R5. Everything else in the harness is retrieval-plus-policy: scores are written to disk and re-injected as prose, or exported later for optional LoRA. This module is the missing MDP:

state  s  — discretized (kind, task, plan, completeness, usable, last, fail)
action a  — tool name, or "final"
reward r  — step: semantic_ok hygiene; terminal: Reward.judge
next   s' — state after the tool result

Each Loop turn is one episode. Transitions land in ~/.pwn/policy_traj.jsonl. Q(s,a) and REINFORCE logits H(s,a) are updated from those tuples and persisted in ~/.pwn/policy.json.

The learned Q values are an ADVISORY term in Registry.rank. They never replace TaskSummarizer planning, plan_first, or CORE_TOOLS. Disable with PWN::Env[:agent][:policy] = false.

Constant Summary collapse

POLICY_FILE = 
File.join(Dir.home, '.pwn', 'policy.json')
TRAJECTORY_FILE = 
File.join(Dir.home, '.pwn', 'policy_traj.jsonl')
ALPHA = 
0.15
ALPHA_PG = 
0.05
GAMMA = 
0.85
EPSILON = 
0.08
STEP_OK = 
0.05
STEP_FAIL = 
-0.20
STEP_TASK = 
0.12
STEP_CLOSED = 
0.20
STEP_GRIND = 
-0.08
MAX_TRAJ = 
2_000
GOLD_MIN = 
0.6
VISITS_MIN = 
2
COLD_EPISODES = 
8
WARM_EPISODES = 
40
TASK_MOD = 
16
ACTION_MOD = 
16
EP_KEY = 
:pwn_policy_episode

Class Method Summary collapse

Class Method Details

.advantage(opts = {}) ⇒ Object

Q(s,a) − V(s). Unknown / cold-start pairs return 0 so rank is unchanged.



395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
 
# File 'lib/pwn/ai/agent/policy.rb', line 395

public_class_method def self.advantage(opts = {})
  return 0.0 unless enabled?

  s = opts[:state] || current_state
  a = opts[:action].to_s
  return 0.0 if s.to_s.empty? || a.empty?

  tab = load
  visits = read_visit(table: tab, state: s, action: a)
  qsa = smoothed_q(table: tab, state: s, action: a)
  # Tiny visit counts stay at 0 unless the value is already decisive.
  return 0.0 if visits < VISITS_MIN && qsa.abs < 0.08

  (qsa - max_q(table: tab, state: s)).round(4)
rescue StandardError
  0.0
end

.attach_episode!(opts = {}) ⇒ Object 



455
456
457
458
 
# File 'lib/pwn/ai/agent/policy.rb', line 455

public_class_method def self.attach_episode!(opts = {})
  Thread.current[EP_KEY] = opts[:episode]
  opts[:episode]
end

.authors ⇒ Object 



651
652
653
 
# File 'lib/pwn/ai/agent/policy.rb', line 651

public_class_method def self.authors
  "AUTHOR(S):\n  0day Inc. <support@0dayinc.com>\n"
end

.begin_episode(opts = {}) ⇒ Object

Supported Method Parameters

ep = PWN::AI::Agent::Policy.begin_episode( session_id: 'optional - PWN::Sessions id', request: 'optional - user request', kind: 'optional - request kind', intent: 'optional - Loop.request_intent', engine: 'optional - active engine', ts_state: 'optional - TaskSummarizer state hash' )



131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'lib/pwn/ai/agent/policy.rb', line 131

public_class_method def self.begin_episode(opts = {})
  return { skipped: :disabled } unless enabled?

  sid = (opts[:session_id] || "ep_#{Thread.current.object_id}").to_s
  task = active_task_text(ts_state: opts[:ts_state], request: opts[:request])
  s0 = state(
    kind: opts[:kind],
    request: task,
    last_action: 'start',
    fails: 0,
    engine: opts[:engine],
    ts_state: opts[:ts_state]
  )
  ep = {
    session_id: sid,
    request: opts[:request].to_s[0, 240],
    kind: normalize_kind(raw: opts[:kind]),
    intent: opts[:intent].to_s,
    engine: opts[:engine].to_s,
    started_at: Time.now.utc.iso8601,
    state: s0,
    last_action: 'start',
    plan_idx: ts_idx(ts_state: opts[:ts_state]),
    plan_open: ts_open?(ts_state: opts[:ts_state]),
    fails: 0,
    steps: []
  }
  Thread.current[EP_KEY] = ep
  ep
rescue StandardError => e
  warn "[pwn-ai/policy] begin_episode swallowed: #{e.class}: #{e.message}"
  nil
end

.cold? ⇒ Boolean

Returns:

  • (Boolean) 


86
87
88
89
90
 
# File 'lib/pwn/ai/agent/policy.rb', line 86

public_class_method def self.cold?
  stats[:n_episodes].to_i < COLD_EPISODES
rescue StandardError
  true
end

.current_episode ⇒ Object 



441
442
443
 
# File 'lib/pwn/ai/agent/policy.rb', line 441

public_class_method def self.current_episode
  Thread.current[EP_KEY]
end

.current_state ⇒ Object 



436
437
438
439
 
# File 'lib/pwn/ai/agent/policy.rb', line 436

public_class_method def self.current_state
  ep = Thread.current[EP_KEY]
  ep.is_a?(Hash) ? ep[:state] : nil
end

.detach_episode! ⇒ Object

Hermes split: snapshot + clear the live episode so Loop.maybe_finish_policy is a no-op on the user-visible path while TurnFinalizer re-attaches it on the background review thread.



449
450
451
452
453
 
# File 'lib/pwn/ai/agent/policy.rb', line 449

public_class_method def self.detach_episode!
  ep = Thread.current[EP_KEY]
  Thread.current[EP_KEY] = nil
  ep
end

.enabled? ⇒ Boolean

Returns:

  • (Boolean) 


638
639
640
641
642
643
644
645
646
647
648
649
 
# File 'lib/pwn/ai/agent/policy.rb', line 638

public_class_method def self.enabled?
  return true unless defined?(PWN::Env) && PWN::Env.is_a?(Hash)

  v = begin
    PWN::Env.dig(:ai, :agent, :policy)
  rescue StandardError
    nil
  end
  v.nil? || !!v
rescue StandardError
  true
end

.episode_budget_met? ⇒ Boolean

True once live returns, warmup-replayed trajectories, or a warmed Q table have enough mass to emit greedy suggestions. Cold? stays a coarser state-encoding gate; this is the banner.

Returns:

  • (Boolean) 


101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/pwn/ai/agent/policy.rb', line 101

public_class_method def self.episode_budget_met?
  n = stats[:n_episodes].to_i
  return true if n >= COLD_EPISODES

  tab = load
  warmed = !tab[:warmed_at].to_s.empty?
  pairs = 0
  tab[:q].each_value { |acts| pairs += acts.length if acts.is_a?(Hash) }
  return true if warmed && pairs >= COLD_EPISODES

  traj_n = trajectories(limit: COLD_EPISODES).length
  warmed && traj_n >= COLD_EPISODES
rescue StandardError
  false
end

.evaluate(opts = {}) ⇒ Object

Replay stored trajectories under the current Q table. Does not write. Used by task 7 (evaluate policy quality).



534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
 
# File 'lib/pwn/ai/agent/policy.rb', line 534

public_class_method def self.evaluate(opts = {})
  rows = trajectories(limit: opts[:limit] || 200)
  return { n: 0, mean_return: nil, greedy_match: nil, mean_abs_td: nil } if rows.empty?

  tab = load
  abs_td = []
  greedy_hits = 0
  greedy_n = 0
  rows.each do |ep|
    Array(ep[:steps]).each do |tr|
      s = tr[:state].to_s
      a = tr[:action].to_s
      next if s.empty? || a.empty?

      qsa = read_q(table: tab, state: s, action: a)
      max_n = tr[:terminal] ? 0.0 : max_q(table: tab, state: tr[:next_state])
      abs_td << (tr[:reward].to_f + (GAMMA * max_n) - qsa).abs
      acts = (tab[:q][s.to_sym] || {}).keys.map(&:to_s)
      next if acts.empty?

      greedy_n += 1
      best = acts.max_by { |act| read_q(table: tab, state: s, action: act) }
      greedy_hits += 1 if best == a
    end
  end
  rets = rows.map { |r| r[:return].to_f }
  {
    n: rows.length,
    mean_return: (rets.sum / rets.length).round(4),
    mean_abs_td: abs_td.empty? ? nil : (abs_td.sum / abs_td.length).round(4),
    greedy_match: greedy_n.positive? ? (greedy_hits.to_f / greedy_n).round(3) : nil
  }
rescue StandardError => e
  { n: 0, error: "#{e.class}: #{e.message}" }
end

.finish(opts = {}) ⇒ Object

Supported Method Parameters

report = PWN::AI::Agent::Policy.finish( session_id: 'optional - active episode id', score: 'optional - Reward.judge 0..1 (training target)', verdict: 'optional - solved|partial|wrong|refused', proxy_ok: 'optional - Boolean fallback when no judge score' )



244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
 
# File 'lib/pwn/ai/agent/policy.rb', line 244

public_class_method def self.finish(opts = {})
  return { skipped: :disabled } unless enabled?

  ep = Thread.current[EP_KEY]
  return { skipped: :no_episode } unless ep.is_a?(Hash)

  return { skipped: :session_mismatch } if opts[:session_id] && ep[:session_id] && opts[:session_id].to_s != ep[:session_id].to_s

  unless ep[:steps].empty?
    last = ep[:steps].last
    last[:next_state] = state(
      kind: ep[:kind],
      request: ep[:request],
      last_action: last[:action] || ep[:last_action],
      fails: ep[:fails],
      engine: ep[:engine],
      ts_state: opts[:ts_state],
      final: opts[:final],
      score: opts[:score]
    )
  end
  terminal = terminal_reward(score: opts[:score], proxy_ok: opts[:proxy_ok])
  if ep[:steps].empty?
    ep[:steps] << {
      state: ep[:state],
      action: 'final',
      reward: 0.0,
      next_state: ep[:state],
      ok: true,
      duration: 0.0,
      terminal: true
    }
  end
  ep[:steps].last[:reward] = (ep[:steps].last[:reward].to_f + terminal).round(4)
  ep[:steps].last[:terminal] = true
  ep[:score] = opts[:score]
  ep[:verdict] = opts[:verdict]
  ep[:return] = discounted_return(steps: ep[:steps])
  ep[:ended_at] = Time.now.utc.iso8601

  n_td = 0
  n_pg = 0
  ep[:steps].each_with_index do |tr, idx|
    n_td += 1 if update_q!(transition: tr)
    g = discounted_return(steps: ep[:steps][idx..])
    n_pg += 1 if update_pg!(state: tr[:state], action: tr[:action], advantage: g - value(state: tr[:state]))
  end

  persist_episode!(episode: ep)
  Thread.current[EP_KEY] = nil
  {
    session_id: ep[:session_id],
    steps: ep[:steps].length,
    return: ep[:return],
    score: opts[:score],
    td_updates: n_td,
    pg_updates: n_pg
  }
rescue StandardError => e
  warn "[pwn-ai/policy] finish swallowed: #{e.class}: #{e.message}"
  Thread.current[EP_KEY] = nil
  { error: "#{e.class}: #{e.message}" }
end

.help ⇒ Object 



655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
 
# File 'lib/pwn/ai/agent/policy.rb', line 655

public_class_method def self.help
  puts <<~USAGE
    USAGE:
      PWN::AI::Agent::Policy.begin_episode(session_id:, request:, kind:, engine:)
      PWN::AI::Agent::Policy.observe_step(action: 'shell', ok: true, duration: 0.2)
      PWN::AI::Agent::Policy.finish(session_id:, score: 0.8, verdict: :solved)
      PWN::AI::Agent::Policy.q(state:, action:)
      PWN::AI::Agent::Policy.advantage(state:, action:)   # Registry.rank term
      PWN::AI::Agent::Policy.recommend(actions: %w[shell pwn_eval])
      PWN::AI::Agent::Policy.evaluate(limit: 100)
      PWN::AI::Agent::Policy.stats
      PWN::AI::Agent::Policy.episode_budget_met?
      PWN::AI::Agent::Policy.to_context
      PWN::AI::Agent::Policy.lean!(dry_run: true)
      PWN::AI::Agent::Policy.warmup!(limit: 200)
      PWN::AI::Agent::Policy.reset

      #{self}.authors
  USAGE
end

.lean!(opts = {}) ⇒ Object 



605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
 
# File 'lib/pwn/ai/agent/policy.rb', line 605

public_class_method def self.lean!(opts = {})
  dry = opts[:dry_run] ? true : false
  return { skipped: true } unless File.exist?(TRAJECTORY_FILE)

  rows = File.readlines(TRAJECTORY_FILE)
  keep = []
  rows.each do |line|
    ep = JSON.parse(line, symbolize_names: true)
    gold = ep[:return].to_f >= GOLD_MIN || ep[:score].to_f >= GOLD_MIN
    keep << [ep, line, gold]
  rescue StandardError
    next
  end
  gold = keep.select { |_, _, g| g }.map { |_, line, _| line }
  rest = keep.reject { |_, _, g| g }.last([MAX_TRAJ - gold.length, 0].max).map { |_, line, _| line }
  out = gold + rest
  unless dry
    tmp = "#{TRAJECTORY_FILE}.#{Process.pid}.tmp"
    File.write(tmp, out.join)
    File.rename(tmp, TRAJECTORY_FILE)
  end
  { removed: rows.length - out.length, remaining: out.length, dry_run: dry }
rescue StandardError => e
  { error: "#{e.class}: #{e.message}" }
end

.load ⇒ Object

Persistence / eval



464
465
466
467
468
469
470
471
472
473
474
475
476
 
# File 'lib/pwn/ai/agent/policy.rb', line 464

public_class_method def self.load
  FileUtils.mkdir_p(File.dirname(POLICY_FILE))
  return blank_table unless File.exist?(POLICY_FILE)

  data = JSON.parse(File.read(POLICY_FILE), symbolize_names: true)
  data[:q] = {} unless data[:q].is_a?(Hash)
  data[:h] = {} unless data[:h].is_a?(Hash)
  data[:visits] = {} unless data[:visits].is_a?(Hash)
  data[:returns] = Array(data[:returns])
  data
rescue StandardError
  blank_table
end

.maybe_warmup! ⇒ Object 



967
968
969
970
971
972
973
974
975
976
977
978
979
 
# File 'lib/pwn/ai/agent/policy.rb', line 967

public_class_method def self.maybe_warmup!
  return { skipped: :disabled } unless enabled?
  return { skipped: :no_traj } unless File.exist?(TRAJECTORY_FILE)

  tab = load
  pairs = 0
  tab[:q].each_value { |acts| pairs += acts.length if acts.is_a?(Hash) }
  return { skipped: :warmed } if !tab[:warmed_at].to_s.empty? && pairs >= COLD_EPISODES && !cold?

  warmup!
rescue StandardError
  { skipped: :error }
end

.observe_step(opts = {}) ⇒ Object

Supported Method Parameters

step = PWN::AI::Agent::Policy.observe_step( session_id: 'optional - must match begin_episode when set', action: 'required - tool name', ok: 'required - Boolean, Reward.semantic_ok', duration: 'optional - Float seconds', ts_state: 'optional - TaskSummarizer state', request: 'optional - used if episode was not begun', kind: 'optional', engine: 'optional' )



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/pwn/ai/agent/policy.rb', line 177

public_class_method def self.observe_step(opts = {})
  return { skipped: :disabled } unless enabled?

  action = opts[:action].to_s
  return { skipped: :no_action } if action.empty?

  ep = Thread.current[EP_KEY]
  if ep.nil? || (opts[:session_id] && ep[:session_id] && opts[:session_id].to_s != ep[:session_id].to_s)
    begin_episode(
      session_id: opts[:session_id],
      request: opts[:request],
      kind: opts[:kind],
      engine: opts[:engine],
      ts_state: opts[:ts_state]
    )
    ep = Thread.current[EP_KEY]
  end
  return { skipped: :no_episode } unless ep.is_a?(Hash)

  ok = opts[:ok] ? true : false
  ep[:fails] = ep[:fails].to_i + 1 unless ok
  distrust = 0.0
  distrust = Reward.proxy_distrust.to_f if defined?(Reward) && Reward.respond_to?(:proxy_distrust)
  reward = if distrust >= 0.85
             0.0
           else
             ok ? STEP_OK : STEP_FAIL
           end
  reward = (reward + english_step_bonus(ep: ep, ts_state: opts[:ts_state], ok: ok, action: action)).round(4)
  ep[:plan_idx] = ts_idx(ts_state: opts[:ts_state])
  ep[:plan_open] = ts_open?(ts_state: opts[:ts_state])
  s = ep[:state]
  task = active_task_text(ts_state: opts[:ts_state], request: ep[:request])
  s2 = state(
    kind: ep[:kind],
    request: task,
    last_action: action,
    fails: ep[:fails],
    engine: ep[:engine],
    ts_state: opts[:ts_state]
  )
  trans = {
    state: s,
    action: action,
    reward: reward,
    next_state: s2,
    ok: ok,
    duration: opts[:duration].to_f,
    terminal: false
  }
  ep[:steps] << trans
  ep[:state] = s2
  ep[:last_action] = action
  trans
rescue StandardError => e
  warn "[pwn-ai/policy] observe_step swallowed: #{e.class}: #{e.message}"
  nil
end

.q(opts = {}) ⇒ Object

Query — used by Registry.rank (advisory only)



381
382
383
384
385
 
# File 'lib/pwn/ai/agent/policy.rb', line 381

public_class_method def self.q(opts = {})
  read_q(table: load, state: opts[:state], action: opts[:action])
rescue StandardError
  0.0
end

.recommend(opts = {}) ⇒ Object

Supported Method Parameters

pick = PWN::AI::Agent::Policy.recommend( state: 'optional - default current episode state', actions: 'required - Array of tool names', epsilon: 'optional - explore probability (default EPSILON)' )



420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
 
# File 'lib/pwn/ai/agent/policy.rb', line 420

public_class_method def self.recommend(opts = {})
  actions = Array(opts[:actions]).map(&:to_s).reject(&:empty?)
  return { action: nil, reason: :empty } if actions.empty?

  s = opts[:state] || current_state || 'unknown'
  eps = opts.key?(:epsilon) ? opts[:epsilon].to_f : EPSILON
  return { action: actions.sample, reason: :explore, state: s, epsilon: eps } if rand < eps

  tab = load
  scored = actions.map { |a| [a, smoothed_q(table: tab, state: s, action: a)] }
  best = scored.max_by { |_, v| v }
  { action: best[0], q: best[1].round(4), reason: :greedy, state: s, ranked: scored.sort_by { |_, v| -v } }
rescue StandardError => e
  { action: Array(opts[:actions]).first, reason: :error, error: e.message }
end

.reset ⇒ Object 



631
632
633
634
635
636
 
# File 'lib/pwn/ai/agent/policy.rb', line 631

public_class_method def self.reset
  FileUtils.rm_f(POLICY_FILE)
  FileUtils.rm_f(TRAJECTORY_FILE)
  Thread.current[EP_KEY] = nil
  blank_table
end

.save(opts = {}) ⇒ Object 



478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
 
# File 'lib/pwn/ai/agent/policy.rb', line 478

public_class_method def self.save(opts = {})
  table = opts[:table] || load
  table[:updated_at] = Time.now.utc.iso8601
  FileUtils.mkdir_p(File.dirname(POLICY_FILE))
  path = POLICY_FILE
  tmp = File.join(File.dirname(path), ".#{File.basename(path)}.#{Process.pid}.tmp")
  File.open(tmp, File::WRONLY | File::CREAT | File::TRUNC, 0o644) do |f|
    f.flock(File::LOCK_EX)
    f.write(JSON.pretty_generate(table))
    f.flush
    f.fsync
  end
  File.rename(tmp, path)
  table
ensure
  FileUtils.rm_f(tmp) if defined?(tmp) && tmp && File.exist?(tmp)
end

.state(opts = {}) ⇒ Object

Supported Method Parameters

key = PWN::AI::Agent::Policy.state( kind: 'optional - statement|question|autonomous_goal|…', request: 'optional - user text / active English task', last_action: 'optional - previous tool name', fails: 'optional - in-turn failure count', engine: 'optional - active engine' )



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/pwn/ai/agent/policy.rb', line 64

public_class_method def self.state(opts = {})
  kind = normalize_kind(raw: opts[:kind] || opts[:request_kind])
  task = task_family(text: opts[:task] || opts[:request])
  eng  = opts[:engine].to_s.empty? ? 'any' : opts[:engine].to_s.downcase
  plan_q = plan_quality_bin(ts_state: opts[:ts_state])
  comp = completeness_bin(final: opts[:final], score: opts[:score])
  use = usable_bin(final: opts[:final], score: opts[:score])
  # Three independent bins so Q can see plan quality, answer
  # completeness, and whether the human actually got a usable result.
  qual = "p#{plan_q}c#{comp}u#{use}"
  if warm?
    last = action_bucket(name: opts[:last_action] || opts[:last] || 'start')
    fail = fail_bin(count: opts[:fails] || opts[:fail_n])
    "#{kind}|#{task}|a#{last}|f#{fail}|#{qual}|#{eng}"
  elsif !cold?
    fail = fail_bin(count: opts[:fails] || opts[:fail_n])
    "#{kind}|#{task}|f#{fail}|#{qual}|#{eng}"
  else
    "#{kind}|#{task}|#{qual}|#{eng}"
  end
end

.stats ⇒ Object 



509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
 
# File 'lib/pwn/ai/agent/policy.rb', line 509

public_class_method def self.stats
  tab = load
  q_pairs = 0
  tab[:q].each_value { |acts| q_pairs += acts.length if acts.is_a?(Hash) }
  rets = Array(tab[:returns])
  mean_r = rets.empty? ? nil : (rets.sum.to_f / rets.length).round(4)
  n_up = tab[:n_updates].to_i
  td_mean = n_up.positive? ? (tab[:td_abs_sum].to_f / n_up).round(4) : nil
  {
    enabled: enabled?,
    n_updates: n_up,
    n_states: tab[:q].length,
    n_pairs: q_pairs,
    n_episodes: rets.length,
    mean_return: mean_r,
    mean_abs_td: td_mean,
    alpha: ALPHA,
    gamma: GAMMA,
    epsilon: EPSILON
  }
end

.to_context(opts = {}) ⇒ Object 



570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
 
# File 'lib/pwn/ai/agent/policy.rb', line 570

public_class_method def self.to_context(opts = {})
  return '' unless enabled?

  maybe_warmup! unless episode_budget_met?
  s = stats
  lines = []
  unless episode_budget_met?
    lines << 'POLICY (R5 tabular Q / REINFORCE — advisory only, does not replace planning)'
    lines << "  policy cold episodes=#{s[:n_episodes]}/#{COLD_EPISODES} — omit greedy suggestion"
    return "#{lines.join("\n")}\n"
  end

  ev = evaluate(limit: opts[:limit] || 40)
  fallback = %w[shell pwn_eval memory_recall mistakes_record mistakes_resolve learning_note_outcome memory_remember]
  pref = begin
    intent = Thread.current[:pwn_request_intent]
    PWN::AI::Agent::Registry.preference_order(intent: intent)
  rescue StandardError
    []
  end
  actions = pref.empty? ? fallback : pref
  rec = begin
    recommend(actions: actions, epsilon: 0.0)[:action]
  rescue StandardError
    nil
  end
  lines << 'POLICY (R5 tabular Q / REINFORCE — advisory only, does not replace planning)'
  lines << "  episodes=#{s[:n_episodes]} states=#{s[:n_states]} pairs=#{s[:n_pairs]} updates=#{s[:n_updates]}"
  lines << "  mean_return=#{s[:mean_return] || '-'} mean|TD|=#{s[:mean_abs_td] || '-'} greedy_match=#{ev[:greedy_match] || '-'}"
  lines << "  current_state=#{current_state || '(none)'} suggest=#{rec || '-'} tool_preference=#{actions.join(',')}"
  "#{lines.join("\n")}\n"
rescue StandardError
  ''
end

.trajectories(opts = {}) ⇒ Object 



496
497
498
499
500
501
502
503
504
505
506
507
 
# File 'lib/pwn/ai/agent/policy.rb', line 496

public_class_method def self.trajectories(opts = {})
  limit = (opts[:limit] || 50).to_i
  return [] unless File.exist?(TRAJECTORY_FILE)

  File.readlines(TRAJECTORY_FILE).last(limit).filter_map do |line|
    JSON.parse(line, symbolize_names: true)
  rescue StandardError
    nil
  end.reverse
rescue StandardError
  []
end

.update_pg!(opts = {}) ⇒ Object

Supported Method Parameters

h = PWN::AI::Agent::Policy.update_pg!( state: 'required', action: 'required', advantage: 'required - scalar G_t − V(s)' )



352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
 
# File 'lib/pwn/ai/agent/policy.rb', line 352

public_class_method def self.update_pg!(opts = {})
  return nil unless enabled?

  s = opts[:state].to_s
  a = opts[:action].to_s
  adv = opts[:advantage].to_f
  return nil if s.empty? || a.empty?
  return 0.0 if adv.abs < 1e-9

  tab = load
  logits = (tab[:h][s.to_sym] || {}).dup
  logits[a.to_sym] = logits[a.to_sym].to_f
  pi = softmax(logits: logits)
  logits.each_key do |act|
    grad = act.to_s == a ? (1.0 - pi[act].to_f) : -pi[act].to_f
    logits[act] = logits[act].to_f + (ALPHA_PG * adv * grad)
  end
  tab[:h][s.to_sym] = logits
  save(table: tab)
  logits[a.to_sym].to_f.round(5)
rescue StandardError => e
  warn "[pwn-ai/policy] update_pg! swallowed: #{e.class}: #{e.message}"
  nil
end

.update_q!(opts = {}) ⇒ Object

Supported Method Parameters

q = PWN::AI::Agent::Policy.update_q!( transition: 'required - Hash with :state :action :reward :next_state :terminal' )



317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
 
# File 'lib/pwn/ai/agent/policy.rb', line 317

public_class_method def self.update_q!(opts = {})
  return nil unless enabled?

  tr = opts[:transition] || opts
  s  = tr[:state].to_s
  a  = tr[:action].to_s
  return nil if s.empty? || a.empty?

  r  = tr[:reward].to_f
  s2 = tr[:next_state].to_s
  term = tr[:terminal] ? true : false
  tab = load
  qsa = read_q(table: tab, state: s, action: a)
  max_n = term ? 0.0 : max_q(table: tab, state: s2)
  target = r + (GAMMA * max_n)
  td = target - qsa
  new_q = qsa + (ALPHA * td)
  write_q!(table: tab, state: s, action: a, value: new_q)
  bump_visit!(table: tab, state: s, action: a)
  tab[:n_updates] = tab[:n_updates].to_i + 1
  tab[:td_abs_sum] = tab[:td_abs_sum].to_f + td.abs
  save(table: tab)
  new_q.round(5)
rescue StandardError => e
  warn "[pwn-ai/policy] update_q! swallowed: #{e.class}: #{e.message}"
  nil
end

.value(opts = {}) ⇒ Object 



387
388
389
390
391
 
# File 'lib/pwn/ai/agent/policy.rb', line 387

public_class_method def self.value(opts = {})
  max_q(table: load, state: opts[:state])
rescue StandardError
  0.0
end

.warm? ⇒ Boolean

Returns:

  • (Boolean) 


92
93
94
95
96
 
# File 'lib/pwn/ai/agent/policy.rb', line 92

public_class_method def self.warm?
  stats[:n_episodes].to_i >= WARM_EPISODES
rescue StandardError
  false
end

.warmup!(opts = {}) ⇒ Object

Replay stored trajectories into Q so a cold table is not empty advice.



922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
 
# File 'lib/pwn/ai/agent/policy.rb', line 922

public_class_method def self.warmup!(opts = {})
  return { skipped: :disabled } unless enabled?
  return { skipped: :no_traj } unless File.exist?(TRAJECTORY_FILE)

  tab = load
  rows = trajectories(limit: opts[:limit] || 400)
  n = 0
  2.times do
    rows.reverse_each do |ep|
      Array(ep[:steps]).each do |tr|
        s = tr[:state].to_s
        a = tr[:action].to_s
        next if s.empty? || a.empty?

        r = tr[:reward].to_f
        s2 = tr[:next_state].to_s
        term = tr[:terminal] ? true : false
        qsa = read_q(table: tab, state: s, action: a)
        max_n = term ? 0.0 : max_q(table: tab, state: s2)
        target = r + (GAMMA * max_n)
        td = target - qsa
        write_q!(table: tab, state: s, action: a, value: qsa + (ALPHA * td))
        bump_visit!(table: tab, state: s, action: a)
        tab[:n_updates] = tab[:n_updates].to_i + 1
        tab[:td_abs_sum] = tab[:td_abs_sum].to_f + td.abs
        n += 1
      end
    end
  end
  # Credit stored returns toward the episode budget so greedy
  # suggestions are not omitted after a successful replay of a
  # table that never finished COLD_EPISODES live turns.
  rets = Array(tab[:returns])
  need = COLD_EPISODES - rets.length
  if need.positive?
    extras = rows.filter_map { |ep| ep[:return] unless ep[:return].nil? }.first(need)
    tab[:returns] = (rets + extras).last(200)
  end
  tab[:warmed_at] = Time.now.utc.iso8601
  save(table: tab)
  { replayed: rows.length, td_updates: n, n_episodes: Array(load[:returns]).length }
rescue StandardError => e
  { error: "#{e.class}: #{e.message}" }
end