Class: RubyLLM::Contract::Eval::ReportStorage

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/contract/eval/report_storage.rb

Overview

Persists eval reports as history entries and regression baselines.

Instance Method Summary collapse

Constructor Details

#initialize(report:, stats:) ⇒ ReportStorage

Returns a new instance of ReportStorage.



11
12
13
14
 
# File 'lib/ruby_llm/contract/eval/report_storage.rb', line 11

def initialize(report:, stats:)
  @report = report
  @stats = stats
end

Instance Method Details

#baseline_exists?(path: nil, model: nil, reasoning_effort: nil) ⇒ Boolean

Returns:

  • (Boolean) 


50
51
52
 
# File 'lib/ruby_llm/contract/eval/report_storage.rb', line 50

def baseline_exists?(path: nil, model: nil, reasoning_effort: nil)
  File.exist?(path || storage_path(Report::BASELINE_DIR, "json", model: model, reasoning_effort: reasoning_effort))
end

#compare_with_baseline(path: nil, model: nil, reasoning_effort: nil) ⇒ Object

Raises:

  • (ArgumentError) 


37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/ruby_llm/contract/eval/report_storage.rb', line 37

def compare_with_baseline(path: nil, model: nil, reasoning_effort: nil)
  file = path || storage_path(Report::BASELINE_DIR, "json", model: model, reasoning_effort: reasoning_effort)
  raise ArgumentError, "No baseline found at #{file}" unless File.exist?(file)

  baseline_data = JSON.parse(File.read(file), symbolize_names: true)
  validate_baseline!(baseline_data)

  BaselineDiff.new(
    baseline_cases: baseline_data[:cases],
    current_cases: @report.results.map { |result| serialize_case(result) }
  )
end

#eval_history(path: nil, model: nil, reasoning_effort: nil) ⇒ Object 



25
26
27
28
 
# File 'lib/ruby_llm/contract/eval/report_storage.rb', line 25

def eval_history(path: nil, model: nil, reasoning_effort: nil)
  EvalHistory.load(path || storage_path(Report::HISTORY_DIR, "jsonl", model: model,
                                                                      reasoning_effort: reasoning_effort))
end

#save_baseline!(path: nil, model: nil, reasoning_effort: nil) ⇒ Object 



30
31
32
33
34
35
 
# File 'lib/ruby_llm/contract/eval/report_storage.rb', line 30

def save_baseline!(path: nil, model: nil, reasoning_effort: nil)
  file = path || storage_path(Report::BASELINE_DIR, "json", model: model, reasoning_effort: reasoning_effort)
  FileUtils.mkdir_p(File.dirname(file))
  File.write(file, JSON.pretty_generate(serialize_for_baseline))
  file
end

#save_history!(path: nil, model: nil, reasoning_effort: nil) ⇒ Object 



16
17
18
19
20
21
22
23
 
# File 'lib/ruby_llm/contract/eval/report_storage.rb', line 16

def save_history!(path: nil, model: nil, reasoning_effort: nil)
  file = path || storage_path(Report::HISTORY_DIR, "jsonl", model: model, reasoning_effort: reasoning_effort)
  entry = history_entry
  entry[:model] = model if model
  entry[:reasoning_effort] = reasoning_effort if reasoning_effort
  EvalHistory.append(file, entry)
  file
end