Class: Legion::LLM::Router::HealthTracker

Inherits:
Object
  • Object
show all
Includes:
Legion::Logging::Helper
Defined in:
lib/legion/llm/router/health_tracker.rb

Constant Summary collapse

OPEN_PENALTY = 
-50
LATENCY_THRESHOLD_MS = 
5000
LATENCY_PENALTY_STEP = 
-10

Instance Method Summary collapse

Constructor Details

#initialize(window_seconds: 300, failure_threshold: 3, cooldown_seconds: 60) ⇒ HealthTracker

Returns a new instance of HealthTracker.



14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/legion/llm/router/health_tracker.rb', line 14

def initialize(window_seconds: 300, failure_threshold: 3, cooldown_seconds: 60)
  @window_seconds    = window_seconds
  @failure_threshold = failure_threshold
  @cooldown_seconds  = cooldown_seconds

  @circuits       = {}
  @latency_window = {}
  @handlers       = {}
  @mutex          = Mutex.new

  register_default_handlers
end

Instance Method Details

#adjustment(provider, instance: nil, offering_id: nil) ⇒ Object

Returns total priority adjustment for a provider. Combines circuit-breaker penalty and latency penalty. When instance: is given, returns that specific instance’s adjustment. When nil, returns the worst-of across all known instances (most pessimistic).



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/legion/llm/router/health_tracker.rb', line 76

def adjustment(provider, instance: nil, offering_id: nil)
  if instance
    key = instance_key(provider, instance)
    return circuit_adjustment(key) + latency_adjustment(key)
  end

  # Check for known instances — return worst-of if any exist
  instances = known_instances(provider)
  if instances.empty?
    # Backward compat: use provider-level or offering-level key
    key = health_key(provider, offering_id)
    key = provider if offering_id && !tracked?(key) && tracked?(provider)
    return circuit_adjustment(key) + latency_adjustment(key)
  end

  instances.map { |k| circuit_adjustment(k) + latency_adjustment(k) }.min
end

#circuit_state(provider, instance: nil, offering_id: nil) ⇒ Object

Returns :closed, :open, or :half_open. When instance: is given, returns that specific instance’s state. When nil, returns the worst state across all known instances.



97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/legion/llm/router/health_tracker.rb', line 97

def circuit_state(provider, instance: nil, offering_id: nil)
  return circuit_state_for_key(instance_key(provider, instance)) if instance

  # Check for known instances — return worst state if any exist
  instances = known_instances(provider)
  if instances.empty?
    # Backward compat: use provider-level or offering-level key
    key = health_key(provider, offering_id)
    key = provider if offering_id && !tracked?(key) && tracked?(provider)
    return circuit_state_for_key(key)
  end

  worst_circuit_state(instances)
end

#register_handler(signal, &block) ⇒ Object

Register a custom handler for a signal type.



28
29
30
 
# File 'lib/legion/llm/router/health_tracker.rb', line 28

def register_handler(signal, &block)
  @handlers[signal.to_sym] = block
end

#report(provider:, signal:, value:, instance: nil, metadata: {}, offering_id: nil) ⇒ Object

Thread-safe signal intake. Dispatches to the registered handler if one exists. When instance: is given, tracks under “provider/instance”. When instance: is nil, tracks under “provider” (backward compat) or broadcasts to all known instances of that provider.



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/legion/llm/router/health_tracker.rb', line 36

def report(provider:, signal:, value:, instance: nil, metadata: {}, offering_id: nil)
  sym     = signal.to_sym
  handler = @handlers[sym]
  return nil unless handler

  if instance
    # Instance-specific tracking
    payload = build_payload(provider: provider, instance: instance,
                            key: instance_key(provider, instance),
                            offering_id: offering_id, signal: sym,
                            value: value, metadata: metadata)
    @mutex.synchronize { handler.call(payload) }
  else
    # Check if we have tracked instances for this provider; if so, broadcast
    instances = known_instances(provider)
    if instances.empty?
      # No instances tracked — use provider-level key (backward compat)
      payload = build_payload(provider: provider, instance: nil,
                              key: health_key(provider, offering_id),
                              offering_id: offering_id, signal: sym,
                              value: value, metadata: metadata)
      @mutex.synchronize { handler.call(payload) }
    else
      # Broadcast to all known instances of this provider
      @mutex.synchronize do
        instances.each do |inst_key|
          payload = build_payload(provider: provider, instance: nil,
                                  key: inst_key, offering_id: offering_id,
                                  signal: sym, value: value, metadata: metadata)
          handler.call(payload)
        end
      end
    end
  end
end

#reset(provider, instance: nil, offering_id: nil) ⇒ Object

Clears circuit and latency data for a single provider.



113
114
115
116
117
118
119
 
# File 'lib/legion/llm/router/health_tracker.rb', line 113

def reset(provider, instance: nil, offering_id: nil)
  key = instance ? instance_key(provider, instance) : health_key(provider, offering_id)
  @mutex.synchronize do
    @circuits.delete(key)
    @latency_window.delete(key)
  end
end

#reset_all ⇒ Object

Clears all state.



122
123
124
125
126
127
 
# File 'lib/legion/llm/router/health_tracker.rb', line 122

def reset_all
  @mutex.synchronize do
    @circuits.clear
    @latency_window.clear
  end
end