Class: RubyLLM::Contract::Eval::PromptDiffPresenter

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/contract/eval/prompt_diff_presenter.rb

Overview

Renders a prompt diff as a readable console summary.

Constant Summary collapse

VARIANT_LABEL_WIDTH = 
12
TABLE_WIDTH = 
26
CASE_SET_MISMATCH_TITLE = 
"  Case set mismatch (safe_to_switch? = NO):"
INPUT_MISMATCH_TITLE = 
"  Input mismatch (safe_to_switch? = NO):"
EXPECTED_MISMATCH_TITLE = 
"  Expected mismatch (safe_to_switch? = NO):"
REGRESSIONS_TITLE = 
"  Regressions (PASS -> FAIL):"
SCORE_DROPS_TITLE = 
"  Score drops:"
IMPROVEMENTS_TITLE = 
"  Improvements:"
REMOVED_PASSING_TITLE = 
"  Removed (were passing in baseline):"

Instance Method Summary collapse

Constructor Details

#initialize(prompt_diff:, comparator:) ⇒ PromptDiffPresenter

Returns a new instance of PromptDiffPresenter.



18
19
20
21
 
# File 'lib/ruby_llm/contract/eval/prompt_diff_presenter.rb', line 18

def initialize(prompt_diff:, comparator:)
  @prompt_diff = prompt_diff
  @comparator = comparator
end

Instance Method Details



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/ruby_llm/contract/eval/prompt_diff_presenter.rb', line 23

def print_summary(io = $stdout)
  print_header(io)
  print_warning(io, "one side has no evaluated cases (all skipped?)") if @comparator.empty_comparison?
  print_case_set_mismatch(io)
  print_formatted_section(io, INPUT_MISMATCH_TITLE, @comparator.input_mismatches) do |mismatch|
    "#{mismatch[:case]}: inputs differ between candidate and baseline"
  end
  print_formatted_section(io, EXPECTED_MISMATCH_TITLE, @comparator.expected_mismatches) do |mismatch|
    "#{mismatch[:case]}: expected values differ between candidate and baseline"
  end
  print_formatted_section(io, REGRESSIONS_TITLE, @prompt_diff.regressions) do |regression|
    "#{regression[:case]}: was PASS, now FAIL -- #{regression[:detail]}"
  end
  print_formatted_section(io, SCORE_DROPS_TITLE, @comparator.score_regressions) do |regression|
    "#{regression[:case]}: #{regression[:baseline_score]} -> #{regression[:candidate_score]} (#{regression[:delta]})"
  end
  print_formatted_section(io, IMPROVEMENTS_TITLE, @prompt_diff.improvements) do |improvement|
    "#{improvement[:case]}: was FAIL, now PASS"
  end
  print_formatted_section(io, REMOVED_PASSING_TITLE, @prompt_diff.removed_passing_cases, &:to_s)
  io.puts "  Safe to switch: #{@comparator.safe_to_switch? ? "YES" : "NO"}"
end