Class: Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Helpers::MonitoringEngine

Inherits:
Object
  • Object
show all
Defined in:
lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ MonitoringEngine

Returns a new instance of MonitoringEngine.



12
13
14
15
16
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 12

def initialize
  @judgments           = {}
  @calibration         = CalibrationTracker.new
  @domain_calibrations = {}
end

Instance Attribute Details

#calibration ⇒ Object (readonly)

Returns the value of attribute calibration.



10
11
12
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 10

def calibration
  @calibration
end

#domain_calibrations ⇒ Object (readonly)

Returns the value of attribute domain_calibrations.



10
11
12
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 10

def domain_calibrations
  @domain_calibrations
end

#judgments ⇒ Object (readonly)

Returns the value of attribute judgments.



10
11
12
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 10

def judgments
  @judgments
end

Instance Method Details

#average_effort(window: CALIBRATION_WINDOW) ⇒ Object 



83
84
85
86
87
88
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 83

def average_effort(window: CALIBRATION_WINDOW)
  recent = @judgments.values.last(window)
  return 0.0 if recent.empty?

  (recent.sum(&:effort_level) / recent.size).round(10)
end

#calibration_report ⇒ Object 



90
91
92
93
94
95
96
97
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 90

def calibration_report
  domain_reports = @domain_calibrations.transform_values(&:to_h)
  {
    overall:        @calibration.to_h,
    by_domain:      domain_reports,
    total_resolved: @judgments.values.count(&:resolved)
  }
end

#detect_overconfidence ⇒ Object 



75
76
77
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 75

def detect_overconfidence
  @judgments.values.select(&:overconfident?)
end

#detect_underconfidence ⇒ Object 



79
80
81
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 79

def detect_underconfidence
  @judgments.values.select(&:underconfident?)
end

#feeling_of_knowing(domain:, query: nil) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 47

def feeling_of_knowing(domain:, query: nil)
  domain_cal   = domain_calibration_for(domain)
  base_conf    = domain_cal.empty? ? DEFAULT_CONFIDENCE : domain_cal.calibration_score
  richness_mod = query.to_s.split.size * 0.02
  confidence   = [(base_conf + richness_mod).clamp(0.0, 1.0), 1.0].min

  record_judgment(
    type:                 :feeling_of_knowing,
    domain:               domain,
    predicted_confidence: confidence,
    effort:               0.3
  )
end

#judgment_of_learning(domain:, content: nil) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 61

def judgment_of_learning(domain:, content: nil)
  domain_cal  = domain_calibration_for(domain)
  base_conf   = domain_cal.empty? ? DEFAULT_CONFIDENCE : domain_cal.calibration_score
  length_mod  = [(content.to_s.length * 0.001), 0.1].min
  confidence  = (base_conf + length_mod).clamp(0.0, 1.0)

  record_judgment(
    type:                 :judgment_of_learning,
    domain:               domain,
    predicted_confidence: confidence,
    effort:               0.4
  )
end

#monitoring_report ⇒ Object 



99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 99

def monitoring_report
  resolved, unresolved = @judgments.values.partition(&:resolved)

  {
    total_judgments:      @judgments.size,
    resolved_count:       resolved.size,
    unresolved_count:     unresolved.size,
    overconfident_count:  detect_overconfidence.size,
    underconfident_count: detect_underconfidence.size,
    average_effort:       average_effort,
    calibration:          @calibration.to_h,
    domain_count:         @domain_calibrations.size
  }
end

#record_judgment(type:, domain:, predicted_confidence: DEFAULT_CONFIDENCE, effort: 0.5) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 18

def record_judgment(type:, domain:, predicted_confidence: DEFAULT_CONFIDENCE, effort: 0.5)
  judgment = MonitoringJudgment.new(
    judgment_type:        type,
    domain:               domain,
    predicted_confidence: predicted_confidence,
    effort_level:         effort
  )

  @judgments[judgment.id] = judgment
  prune_judgments
  judgment
end

#resolve_judgment(judgment_id:, actual_outcome:) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 31

def resolve_judgment(judgment_id:, actual_outcome:)
  judgment = @judgments[judgment_id]
  return nil unless judgment

  judgment.resolve!(actual: actual_outcome)

  @calibration.add_point(predicted: judgment.predicted_confidence, actual: actual_outcome)

  domain_calibration_for(judgment.domain).add_point(
    predicted: judgment.predicted_confidence,
    actual:    actual_outcome
  )

  judgment
end

#to_h ⇒ Object 



114
115
116
117
118
119
120
121
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/monitoring_engine.rb', line 114

def to_h
  {
    judgment_count: @judgments.size,
    calibration:    @calibration.to_h,
    domain_count:   @domain_calibrations.size,
    average_effort: average_effort
  }
end