Class: RubyLLM::Contract::Eval::BaselineDiff

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/contract/eval/baseline_diff.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(baseline_cases:, current_cases:) ⇒ BaselineDiff

Returns a new instance of BaselineDiff.



9
10
11
12
13
14
15
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 9

def initialize(baseline_cases:, current_cases:)
  @baseline = index_by_name(baseline_cases)
  @current = index_by_name(current_cases)
  @baseline_score = compute_score(baseline_cases)
  @current_score = compute_score(current_cases)
  freeze
end

Instance Attribute Details

#baseline_score ⇒ Object (readonly)

Returns the value of attribute baseline_score.



7
8
9
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 7

def baseline_score
  @baseline_score
end

#current_score ⇒ Object (readonly)

Returns the value of attribute current_score.



7
8
9
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 7

def current_score
  @current_score
end

Instance Method Details

#improved? ⇒ Boolean

Returns:

  • (Boolean) 


58
59
60
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 58

def improved?
  improvements.any?
end

#improvements ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 32

def improvements
  @baseline.filter_map do |name, baseline|
    current = @current[name]
    next unless current
    next unless !baseline[:passed] && current[:passed]

    {
      case: name,
      baseline: { passed: baseline[:passed], score: baseline[:score] },
      current: { passed: current[:passed], score: current[:score] }
    }
  end
end

#new_cases ⇒ Object 



62
63
64
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 62

def new_cases
  (@current.keys - @baseline.keys)
end

#regressed? ⇒ Boolean

Returns:

  • (Boolean) 


50
51
52
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 50

def regressed?
  regressions.any? || removed_passing_cases.any?
end

#regressions ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 17

def regressions
  @baseline.filter_map do |name, baseline|
    current = @current[name]
    next unless current
    next unless baseline[:passed] && !current[:passed]

    {
      case: name,
      baseline: { passed: baseline[:passed], score: baseline[:score] },
      current: { passed: current[:passed], score: current[:score] },
      detail: current[:details]
    }
  end
end

#removed_cases ⇒ Object 



66
67
68
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 66

def removed_cases
  (@baseline.keys - @current.keys)
end

#removed_passing_cases ⇒ Object 



54
55
56
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 54

def removed_passing_cases
  removed_cases.select { |name| @baseline[name]&.dig(:passed) }
end

#score_delta ⇒ Object 



46
47
48
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 46

def score_delta
  (current_score - baseline_score).round(4)
end

#to_s ⇒ Object 



70
71
72
73
74
75
76
77
 
# File 'lib/ruby_llm/contract/eval/baseline_diff.rb', line 70

def to_s
  lines = ["Score: #{baseline_score.round(2)} → #{current_score.round(2)} (#{format_delta})"]
  regressions.each { |r| lines << "  REGRESSED  #{r[:case]}: #{r[:detail]}" }
  improvements.each { |r| lines << "  IMPROVED   #{r[:case]}" }
  new_cases.each { |c| lines << "  NEW        #{c}" }
  removed_cases.each { |c| lines << "  REMOVED    #{c}" }
  lines.join("\n")
end