Class: Legion::Extensions::Llm::Provider

Inherits:
Object
  • Object
show all
Includes:
Cache::Helper, Streaming, Logging::Helper
Defined in:
lib/legion/extensions/llm/provider.rb,
lib/legion/extensions/llm/provider/open_ai_compatible.rb

Overview

Base class for LLM providers.

Defined Under Namespace

Modules: OpenAICompatible

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Streaming

build_on_data_handler, build_stream_callback, build_stream_error_response, error_chunk?, faraday_1?, handle_data, handle_error_chunk, handle_error_event, handle_failed_response, handle_json_error_chunk, handle_parsed_error, handle_sse, handle_stream, json_error_payload?, parse_error_from_json, parse_streaming_error, persist_failed_response_body, persist_failed_response_custom_body?, persist_failed_response_env_body?, process_stream_chunk, raise_partial_streaming_error, stream_response

Constructor Details

#initialize(config) ⇒ Provider

Returns a new instance of Provider.



35
36
37
38
39
 
# File 'lib/legion/extensions/llm/provider.rb', line 35

def initialize(config)
  @config = config.is_a?(Hash) ? HashConfig.new(config) : config
  ensure_configured!
  @connection = Connection.new(self, @config)
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



33
34
35
 
# File 'lib/legion/extensions/llm/provider.rb', line 33

def config
  @config
end

#connection ⇒ Object (readonly)

Returns the value of attribute connection.



33
34
35
 
# File 'lib/legion/extensions/llm/provider.rb', line 33

def connection
  @connection
end

Class Method Details

.assume_models_exist? ⇒ Boolean

Returns:

  • (Boolean) 


516
517
518
 
# File 'lib/legion/extensions/llm/provider.rb', line 516

def assume_models_exist?
  false
end

.capabilities ⇒ Object 



488
489
490
 
# File 'lib/legion/extensions/llm/provider.rb', line 488

def capabilities
  nil
end

.configuration_options ⇒ Object 



496
497
498
 
# File 'lib/legion/extensions/llm/provider.rb', line 496

def configuration_options
  []
end

.configuration_requirements ⇒ Object 



492
493
494
 
# File 'lib/legion/extensions/llm/provider.rb', line 492

def configuration_requirements
  []
end

.configured?(config) ⇒ Boolean

Returns:

  • (Boolean) 


524
525
526
 
# File 'lib/legion/extensions/llm/provider.rb', line 524

def configured?(config)
  configuration_requirements.all? { |req| config.send(req) }
end

.default_tier ⇒ Object 



504
505
506
 
# File 'lib/legion/extensions/llm/provider.rb', line 504

def default_tier
  :frontier
end

.default_transport ⇒ Object 



500
501
502
 
# File 'lib/legion/extensions/llm/provider.rb', line 500

def default_transport
  :http
end

.local? ⇒ Boolean

Returns:

  • (Boolean) 


508
509
510
 
# File 'lib/legion/extensions/llm/provider.rb', line 508

def local?
  false
end

.name ⇒ Object 



480
481
482
 
# File 'lib/legion/extensions/llm/provider.rb', line 480

def name
  to_s.split('::').last
end

.remote? ⇒ Boolean

Returns:

  • (Boolean) 


512
513
514
 
# File 'lib/legion/extensions/llm/provider.rb', line 512

def remote?
  !local?
end

.resolve_model_id(model_id, config: nil) ⇒ Object

rubocop:disable Lint/UnusedMethodArgument



520
521
522
 
# File 'lib/legion/extensions/llm/provider.rb', line 520

def resolve_model_id(model_id, config: nil) # rubocop:disable Lint/UnusedMethodArgument
  model_id
end

.slug ⇒ Object 



484
485
486
 
# File 'lib/legion/extensions/llm/provider.rb', line 484

def slug
  name.downcase
end

Instance Method Details

#api_base ⇒ Object

Raises:

  • (NotImplementedError) 


41
42
43
 
# File 'lib/legion/extensions/llm/provider.rb', line 41

def api_base
  raise NotImplementedError
end

#assume_models_exist? ⇒ Boolean

Returns:

  • (Boolean) 


248
249
250
 
# File 'lib/legion/extensions/llm/provider.rb', line 248

def assume_models_exist?
  self.class.assume_models_exist?
end

#cache_control_prefix_tokens ⇒ Object 



232
233
234
235
236
237
238
 
# File 'lib/legion/extensions/llm/provider.rb', line 232

def cache_control_prefix_tokens
  if config.respond_to?(:cache_control_prefix_tokens) && config.cache_control_prefix_tokens
    config.cache_control_prefix_tokens
  else
    4
  end
end

#cache_enabled? ⇒ Boolean

Returns:

  • (Boolean) 


226
227
228
229
230
 
# File 'lib/legion/extensions/llm/provider.rb', line 226

def cache_enabled?
  return false unless config.respond_to?(:llm_cache_enabled)

  config.llm_cache_enabled == true
end

#cache_instance_key ⇒ Object 



463
464
465
466
467
468
469
470
471
 
# File 'lib/legion/extensions/llm/provider.rb', line 463

def cache_instance_key
  if cache_local_instance?
    (respond_to?(:instance_id) ? instance_id : :default).to_s
  else
    require 'digest'
    urls = Array(config_base_url).map { |u| strip_scheme(u).downcase.chomp('/') }.sort
    Digest::SHA256.hexdigest(urls.join('|'))[0, 12]
  end
end

#cache_local_instance? ⇒ Boolean

── Cache helpers with local/shared tier selection ────────────────

Returns:

  • (Boolean) 


427
428
429
430
431
432
 
# File 'lib/legion/extensions/llm/provider.rb', line 427

def cache_local_instance?
  Array(config_base_url).any? do |url|
    host = url.to_s.downcase
    host.include?('localhost') || host.include?('127.0.0.1') || host.include?('::1')
  end
end

#capabilities ⇒ Object 



76
77
78
 
# File 'lib/legion/extensions/llm/provider.rb', line 76

def capabilities
  self.class.capabilities
end

#chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object

rubocop:disable Metrics/ParameterLists



85
86
87
88
 
# File 'lib/legion/extensions/llm/provider.rb', line 85

def chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil,
         tool_prefs: nil)
  complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:)
end

#complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/legion/extensions/llm/provider.rb', line 95

def complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, thinking: nil,
             tool_prefs: nil, &)
  normalized_temperature = maybe_normalize_temperature(temperature, model)
  log_provider_request(
    messages: messages,
    tools: tools,
    temperature: temperature,
    normalized_temperature: normalized_temperature,
    model: model,
    params: params,
    headers: headers,
    schema: schema,
    thinking: thinking,
    tool_prefs: tool_prefs,
    streaming: block_given?
  )

  payload = Utils.deep_merge(
    render_payload(
      messages,
      tools: tools,
      tool_prefs: tool_prefs,
      temperature: normalized_temperature,
      model: model,
      stream: block_given?,
      schema: schema,
      thinking: thinking
    ),
    params
  )

  if block_given?
    stream_response @connection, payload, headers, &
  else
    sync_response @connection, payload, headers
  end
end

#config_base_url ⇒ Object 



385
386
387
 
# File 'lib/legion/extensions/llm/provider.rb', line 385

def config_base_url
  respond_to?(:settings) ? settings[:base_url] : nil
end

#configuration_requirements ⇒ Object 



80
81
82
 
# File 'lib/legion/extensions/llm/provider.rb', line 80

def configuration_requirements
  self.class.configuration_requirements
end

#configured? ⇒ Boolean

Returns:

  • (Boolean) 


222
223
224
 
# File 'lib/legion/extensions/llm/provider.rb', line 222

def configured?
  configuration_requirements.all? { |req| @config.send(req) }
end

#count_tokens(messages:, model:, params: {}) ⇒ Object 



207
208
209
210
211
212
213
 
# File 'lib/legion/extensions/llm/provider.rb', line 207

def count_tokens(messages:, model:, params: {})
  _ = [model, params]
  Array(messages).sum do |message|
    content = message.respond_to?(:content) ? message.content : message[:content] || message['content']
    estimate_text_tokens(content)
  end
end

#discover_offerings(live: false, **filters) ⇒ Object 



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/legion/extensions/llm/provider.rb', line 140

def discover_offerings(live: false, **filters)
  return filter_cached_offerings(Array(@cached_offerings), filters) unless live

  provider_health = health(live:)
  @cached_offerings = Array(list_models(live:, **filters)).filter_map do |model|
    next unless model_matches_filters?(model, filters)
    next unless model_allowed?(model.id)

    offering_from_model(model, health: provider_health)
  end
  @cached_offerings
rescue Faraday::ConnectionFailed => e
  log.warn("[#{slug}] instance=#{provider_instance_id} unreachable: #{e.message}")
  []
end

#embed(text:, model:, dimensions: nil, params: {}, headers: {}) ⇒ Object 



182
183
184
185
186
187
188
 
# File 'lib/legion/extensions/llm/provider.rb', line 182

def embed(text:, model:, dimensions: nil, params: {}, headers: {})
  payload = Utils.deep_merge(render_embedding_payload(text, model:, dimensions:), params)
  response = @connection.post(embedding_url(model:), payload) do |req|
    req.headers = headers.merge(req.headers) unless headers.empty?
  end
  parse_embedding_response(response, model:, text:)
end

#endpoint_manifest ⇒ Object 



274
275
276
277
278
279
280
281
282
283
 
# File 'lib/legion/extensions/llm/provider.rb', line 274

def endpoint_manifest
  endpoint_methods.each_with_object({}) do |(key, method_name), result|
    next unless respond_to?(method_name)

    value = public_send(method_name)
    result[key] = value unless value.nil?
  rescue ArgumentError, NotImplementedError
    next
  end
end

#fetch_model_detail(_model_name) ⇒ Object

Override in subclasses to make a live API call for model detail. Must return a Hash with symbol keys (e.g. { context_window: 128000 }).



459
460
461
 
# File 'lib/legion/extensions/llm/provider.rb', line 459

def fetch_model_detail(_model_name)
  nil
end

#find_reachable_url(urls) ⇒ Object 



397
398
399
400
401
402
403
 
# File 'lib/legion/extensions/llm/provider.rb', line 397

def find_reachable_url(urls)
  urls.each do |url|
    full = normalize_url(url)
    return full if url_reachable?(full)
  end
  nil
end

#format_messages(messages) ⇒ Object 



307
308
309
310
311
312
313
314
 
# File 'lib/legion/extensions/llm/provider.rb', line 307

def format_messages(messages)
  messages.map do |msg|
    {
      role: msg.role.to_s,
      content: msg.content
    }
  end
end

#format_tool_calls(_tool_calls) ⇒ Object 



316
317
318
 
# File 'lib/legion/extensions/llm/provider.rb', line 316

def format_tool_calls(_tool_calls)
  nil
end

#headers ⇒ Object 



45
46
47
 
# File 'lib/legion/extensions/llm/provider.rb', line 45

def headers
  identity_headers
end

#health(live: false) ⇒ Object 



156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/legion/extensions/llm/provider.rb', line 156

def health(live: false)
  readiness_data = readiness(live:)
  raw_health = readiness_data[:health] || readiness_data['health'] || {}
  status = health_status(readiness_data, raw_health)
  {
    provider: slug.to_sym,
    instance_id: provider_instance_id,
    status:,
    ready: readiness_data[:ready] == true || readiness_data['ready'] == true,
    circuit_state: status == 'healthy' ? 'closed' : 'open',
    latency_ms: raw_health[:latency_ms] || raw_health['latency_ms'],
    raw: raw_health
  }.compact
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.health')
  {
    provider: slug.to_sym,
    instance_id: provider_instance_id,
    status: 'unhealthy',
    ready: false,
    circuit_state: 'open',
    error: e.class.name,
    message: e.message
  }
end

#identity_headers ⇒ Object 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/legion/extensions/llm/provider.rb', line 49

def identity_headers
  return {} unless defined?(Legion::Identity::Process) && Legion::Identity::Process.respond_to?(:identity_hash)

  id = Legion::Identity::Process.identity_hash
  hdrs = {
    'x-legion-identity-canonical-name' => id[:canonical_name].to_s,
    'x-legion-identity-trust' => id[:trust].to_s,
    'x-legion-identity-id' => id[:id].to_s,
    'x-legion-identity-kind' => id[:kind].to_s,
    'x-legion-identity-mode' => id[:mode].to_s,
    'x-legion-identity-source' => id[:source].to_s
  }
  hdrs['x-legion-identity-db-principal-id'] = id[:db_principal_id].to_s if id[:db_principal_id]
  hdrs['x-legion-identity-db-identity-id']  = id[:db_identity_id].to_s if id[:db_identity_id]
  hdrs
rescue StandardError
  {}
end

#image(prompt:, model:, size:, with: nil, mask: nil, params: {}) ⇒ Object

rubocop:disable Metrics/ParameterLists



203
204
205
 
# File 'lib/legion/extensions/llm/provider.rb', line 203

def image(prompt:, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Metrics/ParameterLists
  paint(prompt, model:, size:, with:, mask:, params:)
end

#list_models(live: false, **filters) ⇒ Object

rubocop:enable Metrics/ParameterLists



134
135
136
137
138
 
# File 'lib/legion/extensions/llm/provider.rb', line 134

def list_models(live: false, **filters)
  _ = [live, filters]
  response = @connection.get models_url
  parse_list_models_response response, slug, capabilities
end

#local? ⇒ Boolean

Returns:

  • (Boolean) 


240
241
242
 
# File 'lib/legion/extensions/llm/provider.rb', line 240

def local?
  self.class.local?
end

#model_allowed?(model_name) ⇒ Boolean

Returns:

  • (Boolean) 


355
356
357
358
359
360
361
362
363
364
 
# File 'lib/legion/extensions/llm/provider.rb', line 355

def model_allowed?(model_name)
  name = model_name.to_s.downcase
  wl = model_whitelist
  bl = model_blacklist

  return false if wl.any? && wl.none? { |p| name.include?(p) }
  return false if bl.any? && bl.any? { |p| name.include?(p) }

  true
end

#model_blacklist ⇒ Object 



333
334
335
336
337
338
 
# File 'lib/legion/extensions/llm/provider.rb', line 333

def model_blacklist
  bl = config.model_blacklist if config.respond_to?(:model_blacklist)
  bl ||= settings[:model_blacklist] if respond_to?(:settings) && settings.is_a?(Hash)
  bl ||= runtime_provider_setting(:model_blacklist)
  Array(bl).map { |p| p.to_s.downcase }
end

#model_cache_get(key) ⇒ Object 



434
435
436
437
438
439
440
441
 
# File 'lib/legion/extensions/llm/provider.rb', line 434

def model_cache_get(key)
  return nil unless defined?(Legion::Cache)

  cache_local_instance? ? local_cache_get(key) : cache_get(key)
rescue StandardError => e
  handle_exception(e, level: :debug, handled: true, operation: 'llm.provider.model_cache_get', key:)
  nil
end

#model_detail(model_name) ⇒ Object 



443
444
445
446
447
448
449
450
451
452
453
454
455
 
# File 'lib/legion/extensions/llm/provider.rb', line 443

def model_detail(model_name)
  key = model_detail_cache_key(model_name)
  cached = cache_get(key)
  return cached if cached

  result = fetch_model_detail(model_name)
  cache_set(key, result, ttl: 86_400) if result
  result
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.model_detail',
                      model: model_name)
  nil
end

#model_whitelist ⇒ Object

── Model allow-list / deny-list filtering ────────────────────────



326
327
328
329
330
331
 
# File 'lib/legion/extensions/llm/provider.rb', line 326

def model_whitelist
  wl = config.model_whitelist if config.respond_to?(:model_whitelist)
  wl ||= settings[:model_whitelist] if respond_to?(:settings) && settings.is_a?(Hash)
  wl ||= runtime_provider_setting(:model_whitelist)
  Array(wl).map { |p| p.to_s.downcase }
end

#moderate(input, model:) ⇒ Object 



190
191
192
193
194
 
# File 'lib/legion/extensions/llm/provider.rb', line 190

def moderate(input, model:)
  payload = render_moderation_payload(input, model:)
  response = @connection.post moderation_url, payload
  parse_moderation_response(response, model:)
end

#name ⇒ Object 



72
73
74
 
# File 'lib/legion/extensions/llm/provider.rb', line 72

def name
  self.class.name
end

#normalize_url(url) ⇒ Object 



389
390
391
392
393
394
395
 
# File 'lib/legion/extensions/llm/provider.rb', line 389

def normalize_url(url)
  raw = url.to_s.strip
  return raw if raw.match?(%r{^https?://})

  scheme = tls_enabled? ? 'https' : 'http'
  "#{scheme}://#{raw}"
end

#offering_tier ⇒ Object 



372
373
374
 
# File 'lib/legion/extensions/llm/provider.rb', line 372

def offering_tier
  config.respond_to?(:tier) ? config.tier : self.class.default_tier
end

#offering_transport ⇒ Object

── Offering defaults ─────────────────────────────────────────────



368
369
370
 
# File 'lib/legion/extensions/llm/provider.rb', line 368

def offering_transport
  config.respond_to?(:transport) ? config.transport : self.class.default_transport
end

#paint(prompt, model:, size:, with: nil, mask: nil, params: {}) ⇒ Object

rubocop:disable Metrics/ParameterLists



196
197
198
199
200
201
 
# File 'lib/legion/extensions/llm/provider.rb', line 196

def paint(prompt, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Metrics/ParameterLists
  validate_paint_inputs!(with:, mask:)
  payload = render_image_payload(prompt, model:, size:, with:, mask:, params:)
  response = @connection.post images_url(with:, mask:), payload
  parse_image_response(response, model:)
end

#parse_error(response) ⇒ Object 



285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
 
# File 'lib/legion/extensions/llm/provider.rb', line 285

def parse_error(response)
  return if response.body.empty?

  body = try_parse_json(response.body)
  case body
  when Hash
    error = body['error']
    return error if error.is_a?(String)

    body.dig('error', 'message')
  when Array
    body.map do |part|
      error = part['error']
      error.is_a?(String) ? error : part.dig('error', 'message')
    end.join('. ')
  when String
    body[/"message"\s*:\s*"([^"]{1,500})/, 1] || body
  else
    body
  end
end

#parse_tool_calls(_tool_calls) ⇒ Object 



320
321
322
 
# File 'lib/legion/extensions/llm/provider.rb', line 320

def parse_tool_calls(_tool_calls)
  nil
end

#provider_instance_id ⇒ Object 



473
474
475
476
477
 
# File 'lib/legion/extensions/llm/provider.rb', line 473

def provider_instance_id
  return config.instance_id.to_sym if config.respond_to?(:instance_id) && config.instance_id

  :default
end

#readiness(live: false) ⇒ Object 



252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
 
# File 'lib/legion/extensions/llm/provider.rb', line 252

def readiness(live: false)
  metadata = {
    provider: slug.to_sym,
    name: name,
    configured: configured?,
    ready: configured?,
    local: local?,
    remote: remote?,
    api_base: api_base,
    endpoints: endpoint_manifest,
    live: live
  }

  return metadata.merge(health: { checked: false }) unless live && metadata[:endpoints][:health]

  response = @connection.get(metadata[:endpoints][:health])
  metadata.merge(ready: configured? && health_ready?(response.body), health: response.body)
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.readiness')
  metadata.merge(ready: false, health: { error: e.class.name, message: e.message })
end

#remote? ⇒ Boolean

Returns:

  • (Boolean) 


244
245
246
 
# File 'lib/legion/extensions/llm/provider.rb', line 244

def remote?
  self.class.remote?
end

#resolve_base_url ⇒ Object

── Multi-host base_url resolution ────────────────────────────────



378
379
380
381
382
383
 
# File 'lib/legion/extensions/llm/provider.rb', line 378

def resolve_base_url
  urls = Array(config_base_url)
  return nil if urls.empty?

  @resolve_base_url ||= find_reachable_url(urls) || normalize_url(urls.first)
end

#runtime_provider_setting(key) ⇒ Object 



340
341
342
343
344
345
346
347
348
349
350
351
352
353
 
# File 'lib/legion/extensions/llm/provider.rb', line 340

def runtime_provider_setting(key)
  return nil unless defined?(Legion::Settings)

  ext = Legion::Settings[:extensions]
  return nil unless ext.is_a?(Hash) && ext[:llm].is_a?(Hash)

  provider_key = self.class.respond_to?(:slug) ? self.class.slug.to_sym : nil
  return nil unless provider_key

  provider_conf = ext[:llm][provider_key]
  provider_conf.is_a?(Hash) ? provider_conf[key] : nil
rescue StandardError
  nil
end

#slug ⇒ Object 



68
69
70
 
# File 'lib/legion/extensions/llm/provider.rb', line 68

def slug
  self.class.slug
end

#stream_chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object 



90
91
92
93
 
# File 'lib/legion/extensions/llm/provider.rb', line 90

def stream_chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil,
                thinking: nil, tool_prefs: nil, &)
  complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:, &)
end

#strip_scheme(url) ⇒ Object 



405
406
407
 
# File 'lib/legion/extensions/llm/provider.rb', line 405

def strip_scheme(url)
  url.to_s.sub(%r{^https?://}, '')
end

#tls_enabled? ⇒ Boolean

Returns:

  • (Boolean) 


420
421
422
423
 
# File 'lib/legion/extensions/llm/provider.rb', line 420

def tls_enabled?
  tls = respond_to?(:settings) ? settings[:tls] : nil
  tls.is_a?(Hash) && tls[:enabled] == true
end

#transcribe(audio_file, model:, language:) ⇒ Object 



215
216
217
218
219
220
 
# File 'lib/legion/extensions/llm/provider.rb', line 215

def transcribe(audio_file, model:, language:, **)
  file_part = build_audio_file_part(audio_file)
  payload = render_transcription_payload(file_part, model:, language:, **)
  response = @connection.post transcription_url, payload
  parse_transcription_response(response, model:)
end

#url_reachable?(url) ⇒ Boolean

Returns:

  • (Boolean) 


409
410
411
412
413
414
415
416
417
418
 
# File 'lib/legion/extensions/llm/provider.rb', line 409

def url_reachable?(url)
  require 'uri'
  require 'socket'
  uri = URI.parse(url)
  Socket.tcp(uri.host, uri.port, connect_timeout: 1).close
  true
rescue StandardError => e
  handle_exception(e, level: :debug, handled: true, operation: 'llm.provider.url_reachable', url:)
  false
end