Class: RubricLLM::Comparison

Inherits:
Object
  • Object
show all
Defined in:
lib/rubric_llm/comparison.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(report_a, report_b) ⇒ Comparison

Returns a new instance of Comparison.



7
8
9
10
 
# File 'lib/rubric_llm/comparison.rb', line 7

def initialize(report_a, report_b)
  @report_a = report_a
  @report_b = report_b
end

Instance Attribute Details

#report_a ⇒ Object (readonly)

Returns the value of attribute report_a.



5
6
7
 
# File 'lib/rubric_llm/comparison.rb', line 5

def report_a
  @report_a
end

#report_b ⇒ Object (readonly)

Returns the value of attribute report_b.



5
6
7
 
# File 'lib/rubric_llm/comparison.rb', line 5

def report_b
  @report_b
end

Instance Method Details

#results ⇒ Object 



12
13
14
 
# File 'lib/rubric_llm/comparison.rb', line 12

def results
  @results ||= compute_results
end

#significant_improvements(alpha: 0.05) ⇒ Object 



33
34
35
 
# File 'lib/rubric_llm/comparison.rb', line 33

def significant_improvements(alpha: 0.05)
  results.select { |_, r| r[:p_value_adjusted] < alpha && r[:delta].positive? }.keys
end

#significant_regressions(alpha: 0.05) ⇒ Object 



37
38
39
 
# File 'lib/rubric_llm/comparison.rb', line 37

def significant_regressions(alpha: 0.05)
  results.select { |_, r| r[:p_value_adjusted] < alpha && r[:delta].negative? }.keys
end

#summary ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/rubric_llm/comparison.rb', line 16

def summary
  lines = ["A/B Comparison"]
  lines << ("=" * 80)
  lines << "Metric                      A        B    Delta    p-value      p-adj  Sig"
  lines << ("-" * 80)

  results.each do |metric, r|
    lines << format("%-20s %8.3f %8.3f %+8.3f %10.4f %10.4f %4s",
                    metric, r[:mean_a], r[:mean_b], r[:delta], r[:p_value], r[:p_value_adjusted],
                    r[:significance])
  end

  lines << ""
  lines << "p-adj: Holm-Bonferroni adjusted across #{results.size} metrics. Significance uses p-adj."
  lines.join("\n")
end