Class: Legion::LLM::Router

Inherits:
Object
  • Object
show all
Includes:
Legion::LLM::Routing::Filter, Legion::LLM::Routing::Fleet, Legion::LLM::Routing::Outcome, Legion::LLM::Routing::Rank, Legion::Logging::Helper
Defined in:
lib/legion/llm/router.rb

Overview

The per-request routing engine (SSOT v4). The sole routing authority: selection runs against (request, operation, exclusions, the LIVE inventory, one settings generation) and yields one Selection or one typed Rejection.

Constant Summary collapse

OPERATION_BASE =

Base capabilities required by each operation, independent of request shape (reproduced from router/required_capabilities.rb — D10 selection time, this class is the container).

{
  chat:         [].freeze,
  stream_chat:  %i[streaming].freeze,
  embed:        %i[embedding].freeze,
  image:        %i[image].freeze,
  transcribe:   %i[audio_transcription].freeze,
  translate:    [].freeze,
  speak:        %i[audio_speech].freeze,
  moderate:     %i[moderation].freeze,
  count_tokens: [].freeze
}.freeze
TIER_EXTERNAL =

Tiers gated by enterprise privacy mode.

%i[cloud frontier].freeze

Constants included from Legion::LLM::Routing::Outcome

Legion::LLM::Routing::Outcome::RETRYABLE, Legion::LLM::Routing::Outcome::TERMINAL_REJECTION_KIND

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Legion::LLM::Routing::Outcome

#classify_outcome

Methods included from Legion::LLM::Routing::Fleet

#fleet_enabled?, #fleet_lane?

Methods included from Legion::LLM::Routing::Rank

#rank

Methods included from Legion::LLM::Routing::Filter

#body_model_hint_decision_for, #evaluate_capabilities, #evaluate_context, #evaluate_dimensions, #filter_availability, #filter_capability, #filter_context, #filter_embedding_dimensions, #filter_exclusions, #filter_fleet, #filter_instance, #filter_operation, #filter_pins, #filter_policy, #filter_provider, #filter_tier, #filter_type, #filter_weight, #model_policy_for, #preferred_context_range_for

Constructor Details

#initialize(request:, operation:, body_model:) ⇒ Router

Returns a new instance of Router.

Parameters:

  • request (Legion::LLM::Inference::Request) —

    the canonical pipeline request (carries trusted_constraints and the server-created routing_context; routing settings are read from Legion::Settings[:router], not from the request)

  • operation (Symbol) —

    the requested operation (:chat, :stream_chat, :embed, ...)

  • body_model (String, nil) —

    the RAW client-body model value



92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/legion/llm/router.rb', line 92

def initialize(request:, operation:, body_model:, **)
  @request   = request
  @operation = validate_operation!(operation)
  @body_model = body_model

  # Attempt state (absorbed from inference/routing_session.rb — D9).
  @exclusions       = []
  @consumed_targets = {}
  @attempts_used    = 0
  @last_rejection   = nil

  # Request-derived facts — computed ONCE, immutable for this request.
  # Settings come straight from Legion::Settings[:llm][:router] (defaults in
  # settings/router.rb); there is no captured settings-snapshot object.
  trusted = @request.trusted_constraints

  @body_model_hint_decision = body_model_hint_decision_for(
    body_model: @body_model, trusted_model: trusted&.model
  )
  @provider_pin = trusted&.provider&.to_sym
  @instance_pin = trusted&.instance_id
  @tier_pin     = validate_tier!(trusted&.tier)
  @model_pin    = resolve_model_pin(trusted)

  @required_capabilities          = required_capabilities_for(operation: @operation)
  @requested_embedding_dimensions = requested_embedding_dimensions_for
  @input_bound                    = input_bound_for
  @required_output_tokens         = required_output_tokens_for
  @context_budget                 = context_budget_for

  @routing_seed          = validate_seed!(@request.routing_context.routing_seed)
  @maximum_attempts      = positive_int!(trusted&.maximum_attempts || Legion::Settings[:llm][:router][:max_attempts],
                                         :maximum_attempts)
  @affinity_strength_bps = Legion::Settings[:llm][:router][:affinity_strength_bps]
end

Instance Attribute Details

#affinity_strength_bps ⇒ Object (readonly)

Returns the value of attribute affinity_strength_bps.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def affinity_strength_bps
  @affinity_strength_bps
end

#body_model ⇒ Object (readonly)

Returns the value of attribute body_model.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def body_model
  @body_model
end

#body_model_hint_decision ⇒ Object (readonly)

Returns the value of attribute body_model_hint_decision.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def body_model_hint_decision
  @body_model_hint_decision
end

#context_budget ⇒ Object (readonly)

Returns the value of attribute context_budget.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def context_budget
  @context_budget
end

#input_bound ⇒ Object (readonly)

Returns the value of attribute input_bound.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def input_bound
  @input_bound
end

#instance_pin ⇒ Object (readonly)

Returns the value of attribute instance_pin.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def instance_pin
  @instance_pin
end

#last_rejection ⇒ Object (readonly)

Returns the value of attribute last_rejection.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def last_rejection
  @last_rejection
end

#maximum_attempts ⇒ Object (readonly)

Returns the value of attribute maximum_attempts.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def maximum_attempts
  @maximum_attempts
end

#model_pin ⇒ Object (readonly)

Returns the value of attribute model_pin.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def model_pin
  @model_pin
end

#operation ⇒ Object (readonly)

Returns the value of attribute operation.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def operation
  @operation
end

#provider_pin ⇒ Object (readonly)

Returns the value of attribute provider_pin.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def provider_pin
  @provider_pin
end

#request ⇒ Object (readonly)

Returns the value of attribute request.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def request
  @request
end

#requested_embedding_dimensions ⇒ Object (readonly)

Returns the value of attribute requested_embedding_dimensions.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def requested_embedding_dimensions
  @requested_embedding_dimensions
end

#required_capabilities ⇒ Object (readonly)

Returns the value of attribute required_capabilities.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def required_capabilities
  @required_capabilities
end

#required_output_tokens ⇒ Object (readonly)

Returns the value of attribute required_output_tokens.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def required_output_tokens
  @required_output_tokens
end

#routing_seed ⇒ Object (readonly)

Returns the value of attribute routing_seed.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def routing_seed
  @routing_seed
end

#tier_pin ⇒ Object (readonly)

Returns the value of attribute tier_pin.



79
80
81
 
# File 'lib/legion/llm/router.rb', line 79

def tier_pin
  @tier_pin
end

Class Method Details

.privacy_mode? ⇒ Boolean

Enterprise privacy mode (delegates to the shared settings owner; Legion::Settings is a hard dependency — no respond_to? guard).

Returns:

  • (Boolean) 


173
174
175
 
# File 'lib/legion/llm/router.rb', line 173

def self.privacy_mode?(**)
  Legion::Settings.enterprise_privacy?
end

.routing_enabled? ⇒ Boolean

Routing is enabled when at least one instance has a complete publication in the Registry.

Returns:

  • (Boolean) 


145
146
147
148
 
# File 'lib/legion/llm/router.rb', line 145

def self.routing_enabled?(**)
  Legion::Extensions::Llm::Inventory::Registry.snapshot
                                              .each_publication_status.any? { |ps| ps.state == :complete }
end

.tier_available?(tier) ⇒ Boolean

Whether a tier can be used right now. :local / :direct — always available :fleet — available when Legion::Transport is loaded :cloud / :frontier — available unless privacy mode

Returns:

  • (Boolean) 


163
164
165
166
167
168
169
 
# File 'lib/legion/llm/router.rb', line 163

def self.tier_available?(tier, **)
  sym = tier.to_sym
  return false if TIER_EXTERNAL.include?(sym) && privacy_mode?
  return Legion.const_defined?('Transport', false) if sym == :fleet

  true
end

.tier_priority ⇒ Object

The tier priority order. ONE settings spelling: llm.router.tier_priority (default defined in lib/legion/llm/settings/router.rb, so the read never needs a || fallback).



153
154
155
156
157
 
# File 'lib/legion/llm/router.rb', line 153

def self.tier_priority(**)
  Array(Legion::Settings[:llm][:router][:tier_priority]).filter_map do |tier|
    tier.to_sym if tier.respond_to?(:to_sym)
  end
end

Instance Method Details

#add_exclusion(exclusion:) ⇒ Object

Additional request-local exclusion (quota domain, policy, etc.).



235
236
237
238
239
240
 
# File 'lib/legion/llm/router.rb', line 235

def add_exclusion(exclusion:, **)
  @exclusions << exclusion
  log.debug("[llm][router] action=exclusion_added kind=#{exclusion.target_kind} " \
            "reason=#{exclusion.reason}")
  exclusion
end

#apply_global_transition(transition) ⇒ Object

Apply a GlobalTransition (instance_unavailable) from a classification.



273
274
275
276
277
278
279
 
# File 'lib/legion/llm/router.rb', line 273

def apply_global_transition(transition, **)
  Legion::Extensions::Llm::Inventory::Registry.dispatch_instance_unavailable(
    instance_key:       transition.instance_key,
    publisher_token_id: transition.publisher_token_id,
    reason:             transition.reason
  )
end

#attempts_exhausted(snapshot) ⇒ Object

The attempts_exhausted Rejection for the current budget state.



282
283
284
285
286
287
288
 
# File 'lib/legion/llm/router.rb', line 282

def attempts_exhausted(snapshot, **)
  Legion::Extensions::Llm::Routing::Rejection.new(
    kind: :attempts_exhausted,
    reason: "maximum attempts (#{@maximum_attempts}) reached",
    inventory_generation: snapshot.generation, candidate_counts: {}, http_status: 503
  )
end

#attempts_remaining ⇒ Object

Targets remaining before the attempt budget is spent.



182
183
184
 
# File 'lib/legion/llm/router.rb', line 182

def attempts_remaining(**)
  [@maximum_attempts - @attempts_used, 0].max
end

#classify(dispatch_result:, attempt_context:) ⇒ Object

Classify one SelectionDispatch::Result via the Routing::Outcome mixin, apply the returned action's exclusions and (for instance_unavailable) the exact global transition, and return the Action.



245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/legion/llm/router.rb', line 245

def classify(dispatch_result:, attempt_context:, **)
  action = classify_outcome(
    outcome: dispatch_result.outcome, attempt_context: attempt_context,
    attempts_remaining: attempts_remaining
  )
  apply_global_transition(action.global_transition) if action.global_transition
  action.exclusions.each { |ex| add_exclusion(exclusion: ex) }
  log.debug("[llm][router] action=outcome_classified kind=#{dispatch_result.outcome.kind} " \
            "disposition=#{action.disposition} exclusions_added=#{action.exclusions.size}")
  action
end

#consume!(selection) ⇒ Object

Record a consumed selection: consume the target BEFORE any external action, atomically with the attempt-count increment. A consumed tuple cannot be reselected this logical request regardless of lane/generation change.



261
262
263
264
265
266
267
268
269
270
 
# File 'lib/legion/llm/router.rb', line 261

def consume!(selection, **)
  key = selection.attempt_target_key
  @attempts_used += 1
  @consumed_targets[key] = true
  @exclusions << Legion::Extensions::Llm::Routing::Exclusion.new(
    target_kind: :attempt_target, target: key, reason: 'attempt_consumed',
    evidence: { attempt_number: @attempts_used }, lifetime: :request
  )
  log.debug("[llm][router] action=attempt_consumed attempt_number=#{@attempts_used} target=#{key}")
end

#consumed_targets ⇒ Object

Consumed provider+instance+model targets for this request.



134
135
136
 
# File 'lib/legion/llm/router.rb', line 134

def consumed_targets(**)
  @consumed_targets.keys.freeze
end

#exclusions ⇒ Object

Request-local exclusion set (frozen copy — callers never mutate it).



129
130
131
 
# File 'lib/legion/llm/router.rb', line 129

def exclusions(**)
  @exclusions.dup.freeze
end

#next_attempt ⇒ Object

next_lane with the exhaustion guard + attempt-context construction: the executor's hot path. Returns an AttemptContext, or a typed Rejection when the attempt budget is spent or the selection went stale against the live inventory.



204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/legion/llm/router.rb', line 204

def next_attempt(**)
  snapshot = current_inventory
  return attempts_exhausted(snapshot) if @attempts_used >= @maximum_attempts

  selection = selection_for(snapshot)
  return selection if rejection?(selection)

  consume!(selection)

  begin
    Legion::LLM::Inference::AttemptContext.build(
      selection: selection, snapshot: snapshot, attempt_number: @attempts_used
    )
  rescue Legion::LLM::Inference::AttemptContext::Stale => e
    # Target stays consumed; owner captures fresh inventory and retries.
    log.debug("[llm][router] action=stale_selection reason=#{e.message}")
    stale_selection(snapshot)
  end
end

#next_attempt! ⇒ Object

next_attempt that raises Errors::RoutingRejected on a Rejection (the streaming-preflight path — rejection must surface BEFORE the SSE response opens).

Raises:



227
228
229
230
231
232
 
# File 'lib/legion/llm/router.rb', line 227

def next_attempt!(**)
  result = next_attempt
  return result unless rejection?(result)

  raise Legion::LLM::Errors::RoutingRejected.new(rejection: result)
end

#next_lane ⇒ Object

Selection — one Selection or one typed Rejection. Reads the LIVE inventory (never a snapshot passed in): the single source of truth is fetched at the moment of the decision.



196
197
198
 
# File 'lib/legion/llm/router.rb', line 196

def next_lane(**)
  selection_for(current_inventory)
end

#rejection?(value) ⇒ Boolean

True when value is a typed Routing::Rejection (never a Selection).

Returns:

  • (Boolean) 


187
188
189
 
# File 'lib/legion/llm/router.rb', line 187

def rejection?(value, **)
  value.is_a?(Legion::Extensions::Llm::Routing::Rejection)
end

#stale_selection(snapshot) ⇒ Object

The stale_selection Rejection (a Selection no longer valid against fresh inventory).



292
293
294
295
296
297
 
# File 'lib/legion/llm/router.rb', line 292

def stale_selection(snapshot, **)
  Legion::Extensions::Llm::Routing::Rejection.new(
    kind: :stale_selection, reason: 'selected lane drifted from snapshot generation',
    inventory_generation: snapshot.generation, candidate_counts: {}
  )
end