Class: Lemans::Results::Aggregate

Inherits:
Object
  • Object
show all
Defined in:
lib/lemans/results/aggregate.rb

Overview

Rolls trials up the way a leaderboard quotes them: solved out of attempts, median time, mean spend per run. Groups by any 1-3 of task, agent, model — "task-model" reads as two columns.

Constant Summary collapse

KEYS = 
%i[task agent model].freeze
METRICS = 
%i[score time cost steps tokens].freeze
METRIC_SOURCES = 
{ time: :duration_sec, cost: :cost_usd, steps: :steps, tokens: :tokens }.freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(report, keys:) ⇒ Aggregate

Returns a new instance of Aggregate.



24
25
26
27
28
29
30
31
 
# File 'lib/lemans/results/aggregate.rb', line 24

def initialize(report, keys:)
  @report = report
  @keys = keys
  @groups = report.rows
                  .group_by { |row| keys.map { row[_1] } }
                  .map { |values, group| build(values, group) }
                  .sort_by { |group| keys.map { group[_1].to_s } }
end

Instance Attribute Details

#keys ⇒ Object (readonly)

Returns the value of attribute keys.



15
16
17
 
# File 'lib/lemans/results/aggregate.rb', line 15

def keys
  @keys
end

#report ⇒ Object (readonly)

Returns the value of attribute report.



15
16
17
 
# File 'lib/lemans/results/aggregate.rb', line 15

def report
  @report
end

Class Method Details

.keys(spec) ⇒ Object

Raises:



17
18
19
20
21
22
 
# File 'lib/lemans/results/aggregate.rb', line 17

def self.keys(spec)
  keys = spec.to_s.split("-").map(&:to_sym)
  return keys if keys.size.between?(1, 3) && keys.uniq == keys && (keys - KEYS).empty?

  raise ConfigError, "--aggregate: expected 1-3 of #{KEYS.join(", ")} joined by dashes (got #{spec.inspect})"
end

Instance Method Details

#order_by!(column) ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/lemans/results/aggregate.rb', line 33

def order_by!(column)
  column = Sorting.column(column, allowed: keys + METRICS)
  @groups =
    if keys.include?(column)
      Sorting.call(@groups) { _1[column].to_s }
    elsif column == :score
      Sorting.call(@groups, descending: true) { [Rational(_1[:solved], _1[:attempts]), _1[:attempts]] }
    else
      Sorting.call(@groups, descending: true) { _1[METRIC_SOURCES.fetch(column)] }
    end
  self
end

#summary ⇒ Object 



67
 
# File 'lib/lemans/results/aggregate.rb', line 67

def summary = report.summary

#summary_lines ⇒ Object 



69
 
# File 'lib/lemans/results/aggregate.rb', line 69

def summary_lines = report.summary_lines

#to_csv ⇒ Object 



59
60
61
62
63
64
65
 
# File 'lib/lemans/results/aggregate.rb', line 59

def to_csv
  columns = keys + %i[solved attempts duration_sec cost_usd steps tokens]
  CSV.generate do |csv|
    csv << columns
    @groups.each { |group| csv << columns.map { group[_1] } }
  end
end

#to_rows ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/lemans/results/aggregate.rb', line 46

def to_rows
  [keys.map(&:to_s) + METRICS.map(&:to_s)] +
    @groups.map do |group|
      keys.map { |key| display_key(key, group[key]) } + [
        "#{group[:solved]}/#{group[:attempts]}",
        time(group[:duration_sec]),
        cost(group[:cost_usd]),
        mean_display(group[:steps], 1),
        mean_display(group[:tokens], 0)
      ]
    end
end