Class: Legion::Extensions::Agentic::Self::MetacognitiveMonitoring::Helpers::CalibrationTracker

Inherits:
Object
  • Object
show all
Defined in:
lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ CalibrationTracker

Returns a new instance of CalibrationTracker.



12
13
14
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 12

def initialize
  @points = []
end

Instance Attribute Details

#points ⇒ Object (readonly)

Returns the value of attribute points.



10
11
12
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 10

def points
  @points
end

Instance Method Details

#add_point(predicted:, actual:) ⇒ Object 



16
17
18
19
20
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 16

def add_point(predicted:, actual:)
  @points << { predicted: predicted.clamp(0.0, 1.0), actual: actual.clamp(0.0, 1.0) }
  @points.shift while @points.size > MAX_CALIBRATION_POINTS
  self
end

#calibration_curve(bins: 5) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 47

def calibration_curve(bins: 5)
  return [] if @points.empty?

  bin_size  = 1.0 / bins
  bin_edges = Array.new(bins) { |i| (i * bin_size).round(10) }

  bin_edges.map do |edge|
    upper = (edge + bin_size).round(10)
    range = (edge...upper)
    in_bin = @points.select { |p| range.cover?(p[:predicted]) }

    actual_mean = if in_bin.empty?
                    nil
                  else
                    (in_bin.sum { |p| p[:actual] } / in_bin.size).round(10)
                  end

    {
      predicted_range: range,
      sample_count:    in_bin.size,
      actual_accuracy: actual_mean
    }
  end
end

#calibration_label ⇒ Object 



42
43
44
45
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 42

def calibration_label
  score = calibration_score
  CALIBRATION_LABELS.find { |range, _| range.cover?(score) }&.last
end

#calibration_score ⇒ Object 



29
30
31
32
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 29

def calibration_score
  score = (1.0 - mean_calibration_error.abs).clamp(0.0, 1.0)
  score.round(10)
end

#count ⇒ Object 



72
73
74
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 72

def count
  @points.size
end

#empty? ⇒ Boolean

Returns:

  • (Boolean) 


76
77
78
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 76

def empty?
  @points.empty?
end

#mean_calibration_error ⇒ Object 



22
23
24
25
26
27
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 22

def mean_calibration_error
  return 0.0 if @points.empty?

  errors = @points.map { |p| p[:predicted] - p[:actual] }
  (errors.sum / errors.size).round(10)
end

#overconfident? ⇒ Boolean

Returns:

  • (Boolean) 


34
35
36
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 34

def overconfident?
  mean_calibration_error > OVERCONFIDENCE_THRESHOLD
end

#to_h ⇒ Object 



80
81
82
83
84
85
86
87
88
89
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 80

def to_h
  {
    count:                  count,
    mean_calibration_error: mean_calibration_error,
    calibration_score:      calibration_score,
    calibration_label:      calibration_label,
    overconfident:          overconfident?,
    underconfident:         underconfident?
  }
end

#underconfident? ⇒ Boolean

Returns:

  • (Boolean) 


38
39
40
 
# File 'lib/legion/extensions/agentic/self/metacognitive_monitoring/helpers/calibration_tracker.rb', line 38

def underconfident?
  mean_calibration_error < UNDERCONFIDENCE_THRESHOLD
end