Module: Legion::LLM::Fleet::Dispatcher

Extended by:
Legion::Logging::Helper
Defined in:
lib/legion/llm/fleet/dispatcher.rb

Constant Summary collapse

ENVELOPE_KEYS = 
%i[
  app_id caller correlation_id expires_at execution_contract idempotency_key identity
  message_context offering_id operation model priority protocol_version provider
  provider_instance reply_to request_id routing_key signed_token timeout timeout_seconds
  trace_context ttl
].freeze
LEGACY_FIELDS = 
%i[schema_version request_type fleet_correlation_id].freeze

Class Method Summary collapse

Class Method Details

.build_envelope(operation:, request_opts:, message_context:, routing_key: nil, reply_to: nil) ⇒ Object 



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 59

def build_envelope(operation:, request_opts:, message_context:, routing_key: nil, reply_to: nil)
  reject_legacy_fields!(request_opts)
  # Protocol v3 is exact-execution only (06 P2): provider, instance,
  # model, and the exact pair are required — no default fill, no
  # routing decision smuggled into the envelope.
  provider = require_exact_value!(request_opts, :provider)
  provider_instance = require_exact_value!(request_opts, :provider_instance)
  model = require_exact_value!(request_opts, :model)
  execution_contract, offering_id = require_exact_execution!(request_opts)
  timeout = resolve_timeout(operation: operation, override: fetch_option(request_opts, :timeout))
  request_id = next_request_id
  correlation_id = next_request_id
  reply_to ||= ReplyDispatcher.agent_queue_name
  routing_key ||= build_routing_key(
    provider:                provider,
    operation:               operation,
    model:                   model,
    provider_instance:       provider_instance,
    context_window:          context_window_from(request_opts),
    boundary:                fetch_option(request_opts, :network_boundary),
    eligibility_fingerprint: fetch_option(request_opts, :eligibility_fingerprint),
    routing_style:           fetch_option(request_opts, :routing_style)
  )

  envelope = {
    protocol_version:   ::Legion::Extensions::Llm::Fleet::Protocol::VERSION,
    request_id:         request_id,
    correlation_id:     correlation_id,
    idempotency_key:    fetch_option(request_opts, :idempotency_key) || "idem_#{SecureRandom.uuid}",
    operation:          operation,
    provider:           provider,
    provider_instance:  provider_instance,
    model:              model,
    params:             request_params(request_opts),
    routing_key:        routing_key,
    reply_to:           reply_to,
    message_context:    message_context || {},
    caller:             fetch_option(request_opts, :caller) || default_caller,
    trace_context:      fetch_option(request_opts, :trace_context) || {},
    timeout_seconds:    timeout,
    expires_at:         (Time.now.utc + timeout).iso8601,
    ttl:                effective_ttl(request_opts, timeout),
    execution_contract: execution_contract,
    offering_id:        offering_id
  }
  envelope[:signed_token] = dispatch_auth_required? ? TokenIssuer.issue(envelope) : 'unsigned'
  envelope
end

.build_routing_key(provider:, operation:, model:, provider_instance: nil, context_window: nil, boundary: nil, eligibility_fingerprint: nil, routing_style: nil) ⇒ Object 



141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 141

def build_routing_key(provider:, operation:, model:, provider_instance: nil, context_window: nil, boundary: nil,
                      eligibility_fingerprint: nil, routing_style: nil)
  style = routing_style || default_routing_style
  return Lane.offering_key(instance_id: provider_instance || provider, model: model, operation: operation) if style.to_s == 'offering_lane'

  if style.to_s == 'shared_lane'
    return Lane.routing_key(operation: operation, model: model, context_window: context_window,
                            boundary: boundary, eligibility_fingerprint: eligibility_fingerprint)
  end

  "llm.request.#{provider}.#{operation}.#{sanitize_model(model)}"
end

.context_window_from(options) ⇒ Object 



158
159
160
161
162
163
164
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 158

def context_window_from(options)
  limits = fetch_option(options, :limits) || {}
  fetch_option(options, :context_window) ||
    fetch_option(options, :max_context_size) ||
    fetch_option(options, :max_input_tokens) ||
    fetch_option(limits, :context_window)
end

.default_caller ⇒ Object 



321
322
323
324
325
326
327
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 321

def default_caller
  {
    source:       'legion-llm',
    component:    'fleet_dispatcher',
    requested_by: Legion::LLM::PublisherIdentity.requested_by
  }
end

.default_routing_style ⇒ Object 



154
155
156
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 154

def default_routing_style
  Legion::Settings.dig(:llm, :fleet, :dispatch, :routing_style) || :shared_lane
end

.dispatch(operation: nil, request: nil, message_context: {}, routing_key: nil, reply_to: nil, **opts) ⇒ Object

Raises:

  • (ArgumentError) 


28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 28

def dispatch(operation: nil, request: nil, message_context: {}, routing_key: nil, reply_to: nil, **opts)
  operation = normalize_operation(operation || fetch_option(request, :operation) || opts[:operation])
  raise ArgumentError, 'operation is required for fleet dispatch' unless operation

  request_opts = normalize_request(request).merge(opts)
  log.debug "[llm][fleet][dispatcher] action=dispatch.enter operation=#{operation} " \
            "model=#{fetch_option(request_opts, :model)} routing_key=#{routing_key} fleet_available=#{fleet_available?}"
  return error_result('fleet_unavailable', message_context: message_context) unless fleet_available?

  envelope = build_envelope(
    operation:       operation,
    request_opts:    request_opts,
    message_context: message_context,
    routing_key:     routing_key,
    reply_to:        reply_to
  )
  future = register_response(envelope[:correlation_id], expected_delivery(envelope))
  publish_result = publish_request(**envelope)
  unless publish_accepted?(publish_result)
    return publish_error_result(publish_result, envelope[:correlation_id],
                                message_context: message_context)
  end

  wait_for_response(
    envelope[:correlation_id],
    timeout:         envelope[:timeout_seconds],
    message_context: message_context,
    future:          future
  )
end

.dispatch_auth_required? ⇒ Boolean

Returns:

  • (Boolean) 


314
315
316
317
318
319
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 314

def dispatch_auth_required?
  value = Legion::Settings.dig(:llm, :fleet, :dispatch, :require_auth)
  return value != false unless value.nil?

  Legion::Settings.dig(:llm, :fleet, :auth, :require_signed_token) != false
end

.effective_ttl(options, timeout) ⇒ Object 



166
167
168
169
170
171
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 166

def effective_ttl(options, timeout)
  ttl = fetch_option(options, :ttl)
  return ttl if ttl

  fetch_option(options, :expiration_seconds) || timeout
end

.error_result(reason, message_context: {}) ⇒ Object 



304
305
306
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 304

def error_result(reason, message_context: {})
  { success: false, error: reason, message_context: message_context }
end

.expected_delivery(envelope) ⇒ Object 



133
134
135
136
137
138
139
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 133

def expected_delivery(envelope)
  {
    protocol_version: envelope[:protocol_version],
    operation:        envelope[:operation],
    correlation_id:   envelope[:correlation_id]
  }
end

.fetch_option(hash, key) ⇒ Object 



197
198
199
200
201
202
203
204
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 197

def fetch_option(hash, key)
  return nil unless hash.respond_to?(:key?)

  string_key = key.to_s
  return hash[string_key] if hash.key?(string_key)

  hash[key] if hash.key?(key)
end

.fleet_available? ⇒ Boolean

Returns:

  • (Boolean) 


210
211
212
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 210

def fleet_available?
  transport_ready? && fleet_enabled?
end

.fleet_enabled? ⇒ Boolean

Returns:

  • (Boolean) 


218
219
220
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 218

def fleet_enabled?
  Legion::Settings[:llm][:fleet][:dispatch][:enabled] != false
end

.legacy_field_present?(hash, key) ⇒ Boolean

Returns:

  • (Boolean) 


179
180
181
182
183
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 179

def legacy_field_present?(hash, key)
  return false unless hash.respond_to?(:key?)

  hash.key?(key) || hash.key?(key.to_s)
end

.next_request_id ⇒ Object 



233
234
235
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 233

def next_request_id
  "req_#{SecureRandom.uuid}"
end

.normalize_operation(operation) ⇒ Object 



308
309
310
311
312
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 308

def normalize_operation(operation)
  return nil if operation.to_s.empty?

  operation.to_sym
end

.normalize_request(request) ⇒ Object 



189
190
191
192
193
194
195
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 189

def normalize_request(request)
  return {} unless request.respond_to?(:to_h)

  request.to_h.transform_keys do |key|
    key.respond_to?(:to_sym) ? key.to_sym : key
  end
end

.publish_accepted?(publish_result) ⇒ Boolean

Returns:

  • (Boolean) 


250
251
252
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 250

def publish_accepted?(publish_result)
  publish_result.is_a?(Hash) && publish_result[:accepted] == true
end

.publish_error_result(publish_result, correlation_id, message_context: {}) ⇒ Object 



265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 265

def publish_error_result(publish_result, correlation_id, message_context: {})
  ReplyDispatcher.deregister(correlation_id)
  status = publish_result.is_a?(Hash) ? publish_result[:status]&.to_sym : :failed
  error = case status
          when :unroutable
            'no_fleet_queue'
          when :nacked
            'fleet_backpressure'
          when :confirm_timeout
            'fleet_publish_timeout'
          else
            'fleet_publish_failed'
          end
  {
    success:         false,
    error:           error,
    publish_status:  status,
    correlation_id:  correlation_id,
    message_context: message_context
  }
end

.publish_request(**opts) ⇒ Object 



241
242
243
244
245
246
247
248
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 241

def publish_request(**opts)
  log.debug("[llm][fleet][dispatcher] action=publish_request correlation_id=#{opts[:correlation_id]} routing_key=#{opts[:routing_key]}")
  require 'legion/extensions/llm/transport/messages/fleet_request'
  ::Legion::Extensions::Llm::Transport::Messages::FleetRequest.new(**opts).publish(request_publish_options)
rescue StandardError => e
  handle_exception(e, level: :warn, operation: 'llm.fleet.dispatcher.publish_request')
  { accepted: false, status: :failed, error: e.message }
end

.register_response(correlation_id, expected = {}) ⇒ Object 



237
238
239
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 237

def register_response(correlation_id, expected = {})
  ReplyDispatcher.register(correlation_id, expected: expected)
end

.reject_legacy_fields!(request_opts) ⇒ Object 



173
174
175
176
177
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 173

def reject_legacy_fields!(request_opts)
  LEGACY_FIELDS.each do |field|
    raise ArgumentError, "#{field} is not supported by fleet protocol v3" if legacy_field_present?(request_opts, field)
  end
end

.request_params(request_opts) ⇒ Object 



185
186
187
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 185

def request_params(request_opts)
  normalize_request(request_opts).except(*ENVELOPE_KEYS)
end

.request_publish_options ⇒ Object 



254
255
256
257
258
259
260
261
262
263
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 254

def request_publish_options
  dispatch = Legion::Settings[:llm][:fleet][:dispatch]
  {
    mandatory:                  dispatch[:mandatory],
    publisher_confirm:          dispatch[:publisher_confirm],
    publish_confirm_timeout_ms: dispatch[:publish_confirm_timeout_ms] || 500,
    spool:                      dispatch[:spool],
    return_result:              true
  }
end

.require_exact_execution!(request_opts) ⇒ Object

P2: the marker is required and must equal the exact marker; the offering_id must be a nonempty String. Both are signed claims (S2/S3).

Raises:

  • (ArgumentError) 


117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 117

def require_exact_execution!(request_opts)
  execution_contract = fetch_option(request_opts, :execution_contract)
  if execution_contract.nil?
    raise ArgumentError,
          "execution_contract must be #{::Legion::Extensions::Llm::Fleet::Protocol::EXACT_EXECUTION_CONTRACT} (fleet protocol v3)"
  end
  unless execution_contract == ::Legion::Extensions::Llm::Fleet::Protocol::EXACT_EXECUTION_CONTRACT
    raise ArgumentError, "unknown fleet execution_contract marker: #{execution_contract.inspect}"
  end

  offering_id = fetch_option(request_opts, :offering_id)
  raise ArgumentError, 'exact execution contract requires a nonempty String offering_id' unless offering_id.is_a?(String) && !offering_id.strip.empty?

  [execution_contract, offering_id]
end

.require_exact_value!(request_opts, key) ⇒ Object

Raises:

  • (ArgumentError) 


108
109
110
111
112
113
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 108

def require_exact_value!(request_opts, key)
  value = fetch_option(request_opts, key)
  raise ArgumentError, "#{key} is required for fleet protocol v3" if value.nil?

  value
end

.resolve_timeout(operation: :default, override: nil) ⇒ Object 



222
223
224
225
226
227
228
229
230
231
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 222

def resolve_timeout(operation: :default, override: nil)
  return override if override

  op = operation.to_sym
  dispatch = Legion::Settings.dig(:llm, :fleet, :dispatch) || {}
  timeouts = dispatch[:timeouts] || {}
  # N9: the fallback lives in settings (llm.fleet.dispatch.timeout_seconds,
  # default 30) — no inline shadow default at the call site.
  fetch_option(timeouts, op) || dispatch[:timeout_seconds]
end

.sanitize_model(model) ⇒ Object 



206
207
208
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 206

def sanitize_model(model)
  model.to_s.gsub(':', '.')
end

.timeout_result(correlation_id, timeout, message_context: {}) ⇒ Object 



299
300
301
302
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 299

def timeout_result(correlation_id, timeout, message_context: {})
  { success: false, error: 'fleet_timeout', correlation_id: correlation_id,
    timeout: timeout, message_context: message_context }
end

.transport_ready? ⇒ Boolean

Returns:

  • (Boolean) 


214
215
216
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 214

def transport_ready?
  Legion::Settings.dig(:transport, :connected) == true
end

.wait_for_response(correlation_id, timeout:, message_context: {}, future: nil) ⇒ Object 



287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/legion/llm/fleet/dispatcher.rb', line 287

def wait_for_response(correlation_id, timeout:, message_context: {}, future: nil)
  log.debug "[llm][fleet][dispatcher] action=wait_for_response correlation_id=#{correlation_id} timeout=#{timeout}"
  future ||= ReplyDispatcher.register(correlation_id)
  result = future.value!(timeout)
  result || timeout_result(correlation_id, timeout, message_context: message_context)
rescue Concurrent::CancelledOperationError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.fleet.dispatcher.wait_cancelled')
  timeout_result(correlation_id, timeout, message_context: message_context)
ensure
  ReplyDispatcher.deregister(correlation_id)
end