Module: Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Runners::MetacognitiveMonitoring

Includes:
Helpers::Lex
Included in:
Client
Defined in:
lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb

Instance Method Summary collapse

Instance Method Details

#average_effort(window: Helpers::CALIBRATION_WINDOW, engine: nil) ⇒ Object 



124
125
126
127
128
129
130
131
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 124

def average_effort(window: Helpers::CALIBRATION_WINDOW, engine: nil, **)
  eng    = engine || monitoring_engine
  effort = eng.average_effort(window: window)

  label = Helpers::EFFORT_LABELS.find { |range, _| range.cover?(effort) }&.last

  { success: true, average_effort: effort, effort_label: label, window: window }
end

#calibration_curve(bins: 5, engine: nil) ⇒ Object 



133
134
135
136
137
138
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 133

def calibration_curve(bins: 5, engine: nil, **)
  eng   = engine || monitoring_engine
  curve = eng.calibration.calibration_curve(bins: bins)

  { success: true, bins: bins, curve: curve }
end

#calibration_report(engine: nil) ⇒ Object 



106
107
108
109
110
111
112
113
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 106

def calibration_report(engine: nil, **)
  eng    = engine || monitoring_engine
  report = eng.calibration_report

  log.debug "[metacognitive] calibration_report domains=#{report[:by_domain].size}"

  { success: true, report: report }
end

#detect_overconfidence(engine: nil) ⇒ Object 



80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 80

def detect_overconfidence(engine: nil, **)
  eng      = engine || monitoring_engine
  findings = eng.detect_overconfidence

  log.debug "[metacognitive] overconfidence_scan count=#{findings.size}"

  {
    success:  true,
    count:    findings.size,
    findings: findings.map(&:to_h)
  }
end

#detect_underconfidence(engine: nil) ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 93

def detect_underconfidence(engine: nil, **)
  eng      = engine || monitoring_engine
  findings = eng.detect_underconfidence

  log.debug "[metacognitive] underconfidence_scan count=#{findings.size}"

  {
    success:  true,
    count:    findings.size,
    findings: findings.map(&:to_h)
  }
end

#feeling_of_knowing(domain:, query: nil, engine: nil) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 50

def feeling_of_knowing(domain:, query: nil, engine: nil, **)
  eng      = engine || monitoring_engine
  judgment = eng.feeling_of_knowing(domain: domain, query: query)

  log.debug "[metacognitive] fok domain=#{domain} confidence=#{judgment.predicted_confidence.round(2)}"

  {
    success:              true,
    judgment_id:          judgment.id,
    domain:               domain,
    predicted_confidence: judgment.predicted_confidence,
    confidence_label:     judgment.confidence_label
  }
end

#judgment_of_learning(domain:, content: nil, engine: nil) ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 65

def judgment_of_learning(domain:, content: nil, engine: nil, **)
  eng      = engine || monitoring_engine
  judgment = eng.judgment_of_learning(domain: domain, content: content)

  log.debug "[metacognitive] jol domain=#{domain} confidence=#{judgment.predicted_confidence.round(2)}"

  {
    success:              true,
    judgment_id:          judgment.id,
    domain:               domain,
    predicted_confidence: judgment.predicted_confidence,
    confidence_label:     judgment.confidence_label
  }
end

#monitoring_report(engine: nil) ⇒ Object 



115
116
117
118
119
120
121
122
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 115

def monitoring_report(engine: nil, **)
  eng    = engine || monitoring_engine
  report = eng.monitoring_report

  log.debug "[metacognitive] monitoring_report total=#{report[:total_judgments]}"

  { success: true, report: report }
end

#record_judgment(type:, domain:, predicted_confidence: Helpers::DEFAULT_CONFIDENCE, effort: 0.5, engine: nil) ⇒ Object 



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 15

def record_judgment(type:, domain:, predicted_confidence: Helpers::DEFAULT_CONFIDENCE,
                    effort: 0.5, engine: nil, **)
  eng       = engine || monitoring_engine
  type_sym  = type.to_sym

  return { success: false, error: :invalid_judgment_type, valid_types: Helpers::JUDGMENT_TYPES } unless Helpers::JUDGMENT_TYPES.include?(type_sym)

  judgment = eng.record_judgment(
    type:                 type_sym,
    domain:               domain,
    predicted_confidence: predicted_confidence,
    effort:               effort
  )

  log.debug "[metacognitive] record_judgment type=#{type_sym} domain=#{domain} " \
            "confidence=#{judgment.predicted_confidence.round(2)} id=#{judgment.id[0..7]}"

  { success: true, judgment_id: judgment.id, judgment: judgment.to_h }
end

#resolve_judgment(judgment_id:, actual_outcome:, engine: nil) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/runners/metacognitive_monitoring.rb', line 35

def resolve_judgment(judgment_id:, actual_outcome:, engine: nil, **)
  eng      = engine || monitoring_engine
  judgment = eng.resolve_judgment(judgment_id: judgment_id, actual_outcome: actual_outcome)

  unless judgment
    log.debug "[metacognitive] resolve_judgment not_found id=#{judgment_id[0..7]}"
    return { success: false, error: :not_found }
  end

  log.info "[metacognitive] resolved judgment=#{judgment_id[0..7]} " \
           "error=#{judgment.calibration_error&.round(3)}"

  { success: true, judgment_id: judgment_id, judgment: judgment.to_h }
end