Class: Ask::Eval::CostTracker

Inherits:
Object
  • Object
show all
Defined in:
lib/ask/eval/cost_tracker.rb

Overview

Tracks token usage and costs for judge evaluations.

Constant Summary collapse

DEFAULT_PRICING =

Estimated pricing per 1M tokens (USD) for common models. Used when actual pricing isn’t available from the provider.

{
  "gpt-4o-mini" => { input: 0.15, output: 0.60 },
  "gpt-4o" => { input: 2.50, output: 10.00 },
  "gpt-4" => { input: 30.00, output: 60.00 },
  "gpt-3.5-turbo" => { input: 0.50, output: 1.50 },
  "claude-3-5-sonnet" => { input: 3.00, output: 15.00 },
  "claude-3-haiku" => { input: 0.25, output: 1.25 },
  "claude-3-opus" => { input: 15.00, output: 75.00 },
  "gemini-1.5-pro" => { input: 1.25, output: 5.00 },
  "gemini-1.5-flash" => { input: 0.075, output: 0.30 },
  "default" => { input: 1.00, output: 2.00 } # fallback
}.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ CostTracker

Returns a new instance of CostTracker.



24
25
26
27
 
# File 'lib/ask/eval/cost_tracker.rb', line 24

def initialize
  @entries = []
  @mutex = Mutex.new
end

Instance Attribute Details

#entries ⇒ Object (readonly)

Returns the value of attribute entries.



22
23
24
 
# File 'lib/ask/eval/cost_tracker.rb', line 22

def entries
  @entries
end

Instance Method Details

#record(model:, input_tokens: nil, output_tokens: nil, duration: nil) ⇒ Object

Record a single judge call cost.

Parameters:

  • model (String) —

    model identifier

  • input_tokens (Integer, nil) (defaults to: nil) —

    input tokens used

  • output_tokens (Integer, nil) (defaults to: nil) —

    output tokens used

  • duration (Float, nil) (defaults to: nil) —

    elapsed time in seconds



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/ask/eval/cost_tracker.rb', line 34

def record(model:, input_tokens: nil, output_tokens: nil, duration: nil)
  pricing = pricing_for(model)
  input_cost = input_tokens.to_f / 1_000_000 * pricing[:input]
  output_cost = output_tokens.to_f / 1_000_000 * pricing[:output]

  entry = {
    model: model,
    input_tokens: input_tokens,
    output_tokens: output_tokens,
    input_cost: input_cost.round(6),
    output_cost: output_cost.round(6),
    total_cost: (input_cost + output_cost).round(6),
    duration: duration&.round(3)
  }.compact

  @mutex.synchronize { @entries << entry }
end

#reset! ⇒ Object

Reset all tracked data.



85
86
87
 
# File 'lib/ask/eval/cost_tracker.rb', line 85

def reset!
  @mutex.synchronize { @entries.clear }
end

#summary ⇒ Hash

Returns summary of all costs.

Returns:

  • (Hash) —

    summary of all costs



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/ask/eval/cost_tracker.rb', line 53

def summary
  @mutex.synchronize do
    by_judge = @entries.group_by { |e| e[:model] }
    judge_costs = by_judge.transform_values do |entries|
      {
        calls: entries.size,
        total_cost: entries.sum { |e| e[:total_cost] || 0 },
        total_tokens: entries.sum { |e| (e[:input_tokens] || 0) + (e[:output_tokens] || 0) }
      }
    end

    {
      total_cost: @entries.sum { |e| e[:total_cost] || 0 }.round(6),
      total_calls: @entries.size,
      by_judge: judge_costs,
      entries: @entries.dup
    }
  end
end

#to_s ⇒ String

Returns human-readable cost report.

Returns:

  • (String) —

    human-readable cost report



74
75
76
77
78
79
80
81
82
 
# File 'lib/ask/eval/cost_tracker.rb', line 74

def to_s
  s = summary
  lines = ["Cost Report"]
  lines << "  Total: $#{format('%.4f', s[:total_cost])} (#{s[:total_calls]} calls)"
  s[:by_judge].each do |model, data|
    lines << "  #{model}: $#{format('%.4f', data[:total_cost])} (#{data[:calls]} calls, #{data[:total_tokens]} tokens)"
  end
  lines.join("\n")
end