Class: Lemans::Results::Report

Inherits:
Object
  • Object
show all
Defined in:
lib/lemans/results/report.rb

Overview

Reads a runs directory back as a table or CSV. The result files stay the source of truth; unreadable ones are counted and said out loud.

Constant Summary collapse

COLUMNS = 
%i[task agent model reward outcome scored cost_usd steps tokens duration_sec started_at trial tags
detail].freeze
TABLE_COLUMNS = 
%i[task agent model reward outcome cost_usd steps tokens duration_sec trial].freeze
NUMERIC_COLUMNS = 
%i[reward cost_usd steps tokens duration_sec].freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(rows:, unreadable: 0) ⇒ Report

Returns a new instance of Report.



66
67
68
69
 
# File 'lib/lemans/results/report.rb', line 66

def initialize(rows:, unreadable: 0)
  @rows = rows
  @unreadable = unreadable
end

Instance Attribute Details

#rows ⇒ Object (readonly)

Returns the value of attribute rows.



17
18
19
 
# File 'lib/lemans/results/report.rb', line 17

def rows
  @rows
end

#unreadable ⇒ Object (readonly)

Returns the value of attribute unreadable.



17
18
19
 
# File 'lib/lemans/results/report.rb', line 17

def unreadable
  @unreadable
end

Class Method Details

.load(runs_dir, tag: nil) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/lemans/results/report.rb', line 19

def self.load(runs_dir, tag: nil)
  paths = Pathname(runs_dir).glob("**/result.json").sort
  rows = []
  unreadable = 0

  paths.each do |path|
    result = JSON.parse(path.read)
    rows << row_from(result)
  rescue JSON::ParserError, SystemCallError, IOError
    unreadable += 1
  end

  rows = rows.select { _1[:tags].include?(tag) } if tag
  new(rows: rows.sort_by { [_1[:task].to_s, _1[:started_at].to_s] }, unreadable: unreadable)
end

.row_from(result) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/lemans/results/report.rb', line 35

def self.row_from(result)
  {
    task: result["task"],
    agent: result["agent"],
    model: result["model"],
    reward: result["reward"],
    outcome: result.dig("outcome", "name"),
    scored: result.dig("outcome", "scored") == true,
    detail: result.dig("outcome", "detail"),
    cost_usd: result.dig("usage", "cost_usd"),
    steps: result.dig("usage", "steps"),
    tokens: tokens_from(result),
    duration_sec: result["duration_sec"],
    started_at: result["started_at"],
    trial: result["trial"],
    tags: Array(result["tags"]).map(&:to_s)
  }
end

.short_model(model) ⇒ Object

A bench may name no model at all (nop, oracle); the summary needs a label, not a nil for ljust to crash on.



64
 
# File 'lib/lemans/results/report.rb', line 64

def self.short_model(model) = model.to_s.split("/").last || "(default)"

.tokens_from(result) ⇒ Object

Tokens the model actually consumed and produced; cache reads stay out, matching how providers meter a run.



56
57
58
59
60
 
# File 'lib/lemans/results/report.rb', line 56

def self.tokens_from(result)
  input = result.dig("usage", "input_tokens")
  output = result.dig("usage", "output_tokens")
  input.nil? && output.nil? ? nil : input.to_i + output.to_i
end

Instance Method Details

#empty? ⇒ Boolean

Returns:

  • (Boolean) 


71
 
# File 'lib/lemans/results/report.rb', line 71

def empty? = rows.empty? && unreadable.zero?

#order_by!(column) ⇒ Object

Numbers rank best-first the way a leaderboard reads; names sort A-Z. Trials that never measured the column sink to the bottom either way.



75
76
77
78
79
80
 
# File 'lib/lemans/results/report.rb', line 75

def order_by!(column)
  column = Sorting.column(column, allowed: TABLE_COLUMNS)
  descending = NUMERIC_COLUMNS.include?(column)
  @rows = Sorting.call(rows, descending: descending) { _1[column] }
  self
end

#summary ⇒ Object 



82
83
84
 
# File 'lib/lemans/results/report.rb', line 82

def summary
  Tally.call(rows).merge(cost_usd: rows.sum { _1[:cost_usd].to_f })
end

#summary_lines ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/lemans/results/report.rb', line 95

def summary_lines
  per_model = rows.group_by { short_model(_1[:model]) }
  lines =
    if per_model.size > 1
      width = per_model.keys.map(&:length).max
      per_model.map { |model, group| "#{model.ljust(width)}  #{stats(group)}" } +
        ["#{"total".ljust(width)}  #{stats(rows)}"]
    else
      [stats(rows)]
    end
  lines[-1] = "#{lines[-1]} ยท #{unreadable} unreadable result(s) skipped" if unreadable.positive?
  lines
end

#to_csv ⇒ Object 



109
110
111
112
113
114
115
116
 
# File 'lib/lemans/results/report.rb', line 109

def to_csv
  CSV.generate do |csv|
    csv << COLUMNS
    rows.each do |row|
      csv << COLUMNS.map { |column| column == :tags ? Array(row[:tags]).join(" ") : row[column] }
    end
  end
end

#to_rows ⇒ Object 



86
87
88
89
90
91
92
93
 
# File 'lib/lemans/results/report.rb', line 86

def to_rows
  [TABLE_COLUMNS.map(&:to_s)] +
    rows.map do |row|
      TABLE_COLUMNS.map do |column|
        display(column == :model ? short_model(row[:model]) : row[column])
      end
    end
end