Class: RubyLLM::Contract::Eval::ModelComparison

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/contract/eval/model_comparison.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(eval_name:, reports:) ⇒ ModelComparison

Returns a new instance of ModelComparison.



9
10
11
12
13
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 9

def initialize(eval_name:, reports:)
  @eval_name = eval_name
  @reports = reports.dup.freeze # { "model_name" => Report }
  freeze
end

Instance Attribute Details

#eval_name ⇒ Object (readonly)

Returns the value of attribute eval_name.



7
8
9
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 7

def eval_name
  @eval_name
end

#reports ⇒ Object (readonly)

Returns the value of attribute reports.



7
8
9
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 7

def reports
  @reports
end

Instance Method Details

#best_for(min_score: 0.0) ⇒ Object 



27
28
29
30
31
32
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 27

def best_for(min_score: 0.0)
  eligible = @reports.select { |_, report| report.score > 0.0 && report.score >= min_score }
  return nil if eligible.empty?

  eligible.min_by { |_, report| report.total_cost }&.first
end

#cost_for(model) ⇒ Object 



23
24
25
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 23

def cost_for(model)
  @reports[model]&.total_cost
end

#cost_per_point ⇒ Object 



34
35
36
37
38
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 34

def cost_per_point
  @reports.transform_values do |report|
    report.score.positive? ? report.total_cost / report.score : Float::INFINITY
  end
end

#models ⇒ Object 



15
16
17
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 15

def models
  @reports.keys
end

#print_summary(io = $stdout) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 53

def print_summary(io = $stdout)
  io.puts "#{@eval_name} — model comparison"
  io.puts
  io.puts table
  io.puts

  best = best_for(min_score: 0.0)
  io.puts "  Best overall: #{best}" if best

  cheapest_passing = best_for(min_score: 1.0)
  io.puts "  Cheapest at 100%: #{cheapest_passing}" if cheapest_passing
end

#score_for(model) ⇒ Object 



19
20
21
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 19

def score_for(model)
  @reports[model]&.score
end

#table ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 40

def table
  lines = ["  Model                      Score       Cost  Avg Latency"]
  lines << "  #{"-" * 57}"

  @reports.each do |model, report|
    latency = report.avg_latency_ms ? "#{report.avg_latency_ms.round}ms" : "n/a"
    cost = report.total_cost.positive? ? "$#{format("%.4f", report.total_cost)}" : "n/a"
    lines << format("  %-25s %6.2f %10s %12s", model, report.score, cost, latency)
  end

  lines.join("\n")
end

#to_h ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/ruby_llm/contract/eval/model_comparison.rb', line 66

def to_h
  @reports.transform_values do |report|
    {
      score: report.score,
      total_cost: report.total_cost,
      avg_latency_ms: report.avg_latency_ms,
      pass_rate: report.pass_rate,
      passed: report.passed?
    }
  end
end