Class: Legion::CLI::Chat::Tools::ShadowEvalStatus

Inherits:
Tools::Base
  • Object
show all
Defined in:
lib/legion/cli/chat/tools/shadow_eval_status.rb

Class Method Summary collapse

Methods inherited from Tools::Base

deferred, deferred?, description, error_response, extension, handle_exception, input_schema, log, mcp_category, mcp_tier, runner, sticky, tags, text_response, tool_name, trigger_words

Class Method Details

.call(action: 'summary') ⇒ Object 



18
19
20
21
22
23
24
25
 
# File 'lib/legion/cli/chat/tools/shadow_eval_status.rb', line 18

def self.call(action: 'summary')
  return 'Shadow evaluation not available.' unless shadow_available?

  case action.to_s
  when 'history' then format_history
  else format_summary
  end
end

.format_history ⇒ Object 



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/legion/cli/chat/tools/shadow_eval_status.rb', line 58

def self.format_history
  entries = Legion::LLM::ShadowEval.history
  return 'No shadow evaluation history.' if entries.empty?

  lines = [format("Shadow Evaluation History (last %<n>d):\n", n: entries.size)]

  entries.last(10).reverse_each do |entry|
    lines << format(
      '  %<time>s  %<pm>s vs %<sm>s  ratio=%<r>.2f  savings=%<s>.1f%%',
      time: entry[:evaluated_at]&.strftime('%H:%M:%S') || '??:??:??',
      pm:   truncate(entry[:primary_model].to_s, 20),
      sm:   truncate(entry[:shadow_model].to_s, 15),
      r:    entry[:length_ratio],
      s:    (entry[:cost_savings] || 0) * 100
    )
  end

  lines.join("\n")
end

.format_summary ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/legion/cli/chat/tools/shadow_eval_status.rb', line 31

def self.format_summary
  s = Legion::LLM::ShadowEval.summary
  lines = ["Shadow Evaluation Summary:\n"]
  lines << format('  Evaluations:      %<v>d', v: s[:total_evaluations])

  if s[:total_evaluations].zero?
    lines << '  No evaluations recorded yet.'
    lines << ''
    lines << '  Enable via settings: llm.shadow.enabled = true'
    return lines.join("\n")
  end

  lines << format('  Avg Length Ratio:  %<v>.2f', v: s[:avg_length_ratio])
  lines << format('  Avg Cost Savings:  %<v>.1f%%', v: s[:avg_cost_savings] * 100)
  lines << format('  Primary Cost:      $%<v>.6f', v: s[:total_primary_cost])
  lines << format('  Shadow Cost:       $%<v>.6f', v: s[:total_shadow_cost])
  lines << format('  Models Tested:     %<v>s', v: s[:models_evaluated].join(', '))

  if s[:avg_cost_savings].positive?
    lines << ''
    lines << format('  Shadow models saved ~%<v>.1f%% on average.',
                    v: s[:avg_cost_savings] * 100)
  end

  lines.join("\n")
end

.shadow_available? ⇒ Boolean

Returns:

  • (Boolean) 


27
28
29
 
# File 'lib/legion/cli/chat/tools/shadow_eval_status.rb', line 27

def self.shadow_available?
  defined?(Legion::LLM::ShadowEval)
end

.truncate(str, max) ⇒ Object 



78
79
80
 
# File 'lib/legion/cli/chat/tools/shadow_eval_status.rb', line 78

def self.truncate(str, max)
  str.length > max ? "#{str[0, max - 1]}~" : str
end