Class: RubyLLM::Contract::Eval::EvalHistory

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/contract/eval/eval_history.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(runs:) ⇒ EvalHistory

Returns a new instance of EvalHistory.



12
13
14
15
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 12

def initialize(runs:)
  @runs = runs.freeze
  freeze
end

Instance Attribute Details

#runs ⇒ Object (readonly)

Returns the value of attribute runs.



10
11
12
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 10

def runs
  @runs
end

Class Method Details

.append(path, run_data) ⇒ Object 



28
29
30
31
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 28

def self.append(path, run_data)
  FileUtils.mkdir_p(File.dirname(path))
  File.open(path, "a") { |f| f.puts(run_data.to_json) }
end

.load(path) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 17

def self.load(path)
  return new(runs: []) unless File.exist?(path)

  runs = File.readlines(path).filter_map do |line|
    JSON.parse(line.strip, symbolize_names: true)
  rescue JSON::ParserError
    nil
  end
  new(runs: runs)
end

Instance Method Details

#dates ⇒ Object 



59
60
61
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 59

def dates
  runs.map { |r| r[:date] }
end

#drift?(threshold: 0.1) ⇒ Boolean

Returns:

  • (Boolean) 


47
48
49
50
51
52
53
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 47

def drift?(threshold: 0.1)
  return false if runs.length < 2

  baseline_score = runs.first[:score]
  current_score = runs.last[:score]
  (baseline_score - current_score) > threshold
end

#latest ⇒ Object 



63
64
65
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 63

def latest
  runs.last
end

#score_trend ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 33

def score_trend
  return :unknown if runs.length < 2

  scores = runs.map { |r| r[:score] }
  recent = scores.last(3)
  if recent.all? { |s| s >= scores.first }
    :stable_or_improving
  elsif recent.last < scores.max * 0.9
    :declining
  else
    :stable_or_improving
  end
end

#scores ⇒ Object 



55
56
57
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 55

def scores
  runs.map { |r| r[:score] }
end

#to_s ⇒ Object 



67
68
69
70
71
72
73
74
75
 
# File 'lib/ruby_llm/contract/eval/eval_history.rb', line 67

def to_s
  return "No history" if runs.empty?

  lines = ["#{runs.length} runs"]
  runs.last(5).each do |r|
    lines << "  #{r[:date]} score=#{r[:score].round(2)} cost=$#{format("%.6f", r[:total_cost] || r[:cost] || 0)}"
  end
  lines.join("\n")
end