Module: Legion::LLM::Arbitrage

Extended by:
Legion::Logging::Helper
Defined in:
lib/legion/llm/arbitrage.rb

Constant Summary collapse

DEFAULT_COST_TABLE =

Default cost table: per-1M-token input/output prices in USD. Overridable via settings: llm.arbitrage.cost_table

{
  'claude-sonnet-4-6'                 => { input: 3.0, output: 15.0 },
  'us.anthropic.claude-sonnet-4-6-v1' => { input: 3.0, output: 15.0 },
  'gpt-4o'                            => { input: 2.5,  output: 10.0 },
  'gpt-4o-mini'                       => { input: 0.15, output: 0.60 },
  'gemini-2.0-flash'                  => { input: 0.10, output: 0.40 },
  'llama3'                            => { input: 0.0,  output: 0.0  }
}.freeze

Class Method Summary collapse

Class Method Details

.cheapest_for(capability: :moderate, max_cost: nil, input_tokens: 1000, output_tokens: 500) ⇒ String?

Selects the cheapest model that meets the capability and quality floor requirements.

Parameters:

  • capability (String, Symbol) (defaults to: :moderate) —

    required capability tier (e.g., :basic, :moderate, :reasoning)

  • max_cost (Float, nil) (defaults to: nil) —

    maximum acceptable cost per typical request (USD); nil means no limit

  • input_tokens (Integer) (defaults to: 1000) —

    estimated input tokens for cost calculation

  • output_tokens (Integer) (defaults to: 500) —

    estimated output tokens for cost calculation

Returns:

  • (String, nil) —

    cheapest eligible model ID, or nil if none qualify



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/legion/llm/arbitrage.rb', line 46

def cheapest_for(capability: :moderate, max_cost: nil, input_tokens: 1000, output_tokens: 500)
  return nil unless enabled?

  quality_floor = settings.fetch(:quality_floor, 0.7)
  eligible = eligible_models(capability: capability, quality_floor: quality_floor)

  scored = eligible.filter_map do |model|
    cost = estimated_cost(model: model, input_tokens: input_tokens, output_tokens: output_tokens)
    next if cost.nil?
    next if max_cost && cost > max_cost

    [model, cost]
  end

  return nil if scored.empty?

  selected = scored.min_by { |_model, cost| cost }&.first
  log.debug("Arbitrage selected model=#{selected} capability=#{capability}")
  selected
end

.cost_table ⇒ Object

Returns the merged cost table: defaults overridden by any settings-defined entries.



68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/legion/llm/arbitrage.rb', line 68

def cost_table
  overrides = settings.fetch(:cost_table, {})
  return DEFAULT_COST_TABLE if overrides.nil? || overrides.empty?

  merged = DEFAULT_COST_TABLE.dup
  overrides.each do |model, costs|
    entry = costs.transform_keys(&:to_sym)
    merged[model.to_s] = entry
  end
  merged
end

.enabled? ⇒ Boolean

Returns true when arbitrage is enabled in settings.

Returns:

  • (Boolean) 


22
23
24
 
# File 'lib/legion/llm/arbitrage.rb', line 22

def enabled?
  settings.fetch(:enabled, false) == true
end

.estimated_cost(model:, input_tokens: 1000, output_tokens: 500) ⇒ Float?

Returns the estimated cost for a request with the given token counts.

Parameters:

  • model (String) —

    model identifier

  • input_tokens (Integer) (defaults to: 1000) —

    estimated number of input tokens

  • output_tokens (Integer) (defaults to: 500) —

    estimated number of output tokens

Returns:

  • (Float, nil) —

    estimated cost in USD, or nil if model not in table



32
33
34
35
36
37
 
# File 'lib/legion/llm/arbitrage.rb', line 32

def estimated_cost(model:, input_tokens: 1000, output_tokens: 500)
  entry = cost_table[model.to_s]
  return nil if entry.nil?

  ((entry[:input] * input_tokens) + (entry[:output] * output_tokens)) / 1_000_000.0
end