Class: RubyLLM::Contract::Eval::ModelComparison

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/contract/eval/model_comparison.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(eval_name:, reports:, configs: nil) ⇒ ModelComparison

Returns a new instance of ModelComparison.



14
15
16
17
18
19
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 14

def initialize(eval_name:, reports:, configs: nil)
  @eval_name = eval_name
  @reports = reports.dup.freeze
  @configs = (configs || default_configs_from_reports).freeze
  freeze
end

Instance Attribute Details

#configs ⇒ Object (readonly)

Returns the value of attribute configs.



7
8
9
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 7

def configs
  @configs
end

#eval_name ⇒ Object (readonly)

Returns the value of attribute eval_name.



7
8
9
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 7

def eval_name
  @eval_name
end

#reports ⇒ Object (readonly)

Returns the value of attribute reports.



7
8
9
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 7

def reports
  @reports
end

Class Method Details

.candidate_label(config) ⇒ Object 



9
10
11
12
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 9

def self.candidate_label(config)
  effort = config[:reasoning_effort]
  effort ? "#{config[:model]} (effort: #{effort})" : config[:model]
end

Instance Method Details

#best_for(min_score: 0.0) ⇒ Object 



33
34
35
36
37
38
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 33

def best_for(min_score: 0.0)
  eligible = @reports.select { |_, report| report.score > 0.0 && report.score >= min_score }
  return nil if eligible.empty?

  eligible.min_by { |_, report| report.total_cost }&.first
end

#cost_for(candidate) ⇒ Object 



29
30
31
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 29

def cost_for(candidate)
  @reports[resolve_key(candidate)]&.total_cost
end

#cost_per_point ⇒ Object 



40
41
42
43
44
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 40

def cost_per_point
  @reports.transform_values do |report|
    report.score.positive? ? report.total_cost / report.score : Float::INFINITY
  end
end

#models ⇒ Object 



21
22
23
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 21

def models
  @reports.keys
end

#print_summary(io = $stdout) ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 60

def print_summary(io = $stdout)
  io.puts "#{@eval_name} — model comparison"
  io.puts
  io.puts table
  io.puts

  best = best_for(min_score: 0.0)
  io.puts "  Best overall: #{best}" if best

  cheapest_passing = best_for(min_score: 1.0)
  io.puts "  Cheapest at 100%: #{cheapest_passing}" if cheapest_passing
end

#score_for(candidate) ⇒ Object 



25
26
27
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 25

def score_for(candidate)
  @reports[resolve_key(candidate)]&.score
end

#table ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 46

def table
  max_label = [@reports.keys.map(&:length).max || 0, 25].max
  lines = [format("  %-#{max_label}s  Score       Cost  Avg Latency", "Candidate")]
  lines << "  #{"-" * (max_label + 36)}"

  @reports.each do |label, report|
    latency = report.avg_latency_ms ? "#{report.avg_latency_ms.round}ms" : "n/a"
    cost = report.total_cost.positive? ? "$#{format("%.4f", report.total_cost)}" : "n/a"
    lines << format("  %-#{max_label}s %6.2f %10s %12s", label, report.score, cost, latency)
  end

  lines.join("\n")
end

#to_h ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 73

def to_h
  @reports.transform_values do |report|
    {
      score: report.score,
      total_cost: report.total_cost,
      avg_latency_ms: report.avg_latency_ms,
      pass_rate: report.pass_rate,
      pass_rate_ratio: report.pass_rate_ratio,
      passed: report.passed?
    }
  end
end