Class: Legion::Extensions::Llm::Provider

Inherits:
Object
  • Object
show all
Includes:
Cache::Helper, Streaming, Logging::Helper
Defined in:
lib/legion/extensions/llm/provider.rb,
lib/legion/extensions/llm/provider/open_ai_compatible.rb

Overview

Base class for LLM providers.

Defined Under Namespace

Modules: OpenAICompatible

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Streaming

build_on_data_handler, build_stream_callback, build_stream_error_response, error_chunk?, faraday_1?, handle_data, handle_error_chunk, handle_error_event, handle_failed_response, handle_json_error_chunk, handle_parsed_error, handle_sse, handle_stream, json_error_payload?, parse_error_from_json, parse_streaming_error, persist_failed_response_body, persist_failed_response_custom_body?, persist_failed_response_env_body?, process_stream_chunk, raise_partial_streaming_error, stream_response

Constructor Details

#initialize(config) ⇒ Provider

Returns a new instance of Provider.



35
36
37
38
39
 
# File 'lib/legion/extensions/llm/provider.rb', line 35

def initialize(config)
  @config = config.is_a?(Hash) ? HashConfig.new(config) : config
  ensure_configured!
  @connection = Connection.new(self, @config)
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



33
34
35
 
# File 'lib/legion/extensions/llm/provider.rb', line 33

def config
  @config
end

#connection ⇒ Object (readonly)

Returns the value of attribute connection.



33
34
35
 
# File 'lib/legion/extensions/llm/provider.rb', line 33

def connection
  @connection
end

Class Method Details

.assume_models_exist? ⇒ Boolean

Returns:

  • (Boolean) 


448
449
450
 
# File 'lib/legion/extensions/llm/provider.rb', line 448

def assume_models_exist?
  false
end

.capabilities ⇒ Object 



428
429
430
 
# File 'lib/legion/extensions/llm/provider.rb', line 428

def capabilities
  nil
end

.configuration_options ⇒ Object 



436
437
438
 
# File 'lib/legion/extensions/llm/provider.rb', line 436

def configuration_options
  []
end

.configuration_requirements ⇒ Object 



432
433
434
 
# File 'lib/legion/extensions/llm/provider.rb', line 432

def configuration_requirements
  []
end

.configured?(config) ⇒ Boolean

Returns:

  • (Boolean) 


456
457
458
 
# File 'lib/legion/extensions/llm/provider.rb', line 456

def configured?(config)
  configuration_requirements.all? { |req| config.send(req) }
end

.local? ⇒ Boolean

Returns:

  • (Boolean) 


440
441
442
 
# File 'lib/legion/extensions/llm/provider.rb', line 440

def local?
  false
end

.name ⇒ Object 



420
421
422
 
# File 'lib/legion/extensions/llm/provider.rb', line 420

def name
  to_s.split('::').last
end

.remote? ⇒ Boolean

Returns:

  • (Boolean) 


444
445
446
 
# File 'lib/legion/extensions/llm/provider.rb', line 444

def remote?
  !local?
end

.resolve_model_id(model_id, config: nil) ⇒ Object

rubocop:disable Lint/UnusedMethodArgument



452
453
454
 
# File 'lib/legion/extensions/llm/provider.rb', line 452

def resolve_model_id(model_id, config: nil) # rubocop:disable Lint/UnusedMethodArgument
  model_id
end

.slug ⇒ Object 



424
425
426
 
# File 'lib/legion/extensions/llm/provider.rb', line 424

def slug
  name.downcase
end

Instance Method Details

#api_base ⇒ Object

Raises:

  • (NotImplementedError) 


41
42
43
 
# File 'lib/legion/extensions/llm/provider.rb', line 41

def api_base
  raise NotImplementedError
end

#assume_models_exist? ⇒ Boolean

Returns:

  • (Boolean) 


215
216
217
 
# File 'lib/legion/extensions/llm/provider.rb', line 215

def assume_models_exist?
  self.class.assume_models_exist?
end

#cache_instance_key ⇒ Object 



403
404
405
406
407
408
409
410
411
 
# File 'lib/legion/extensions/llm/provider.rb', line 403

def cache_instance_key
  if cache_local_instance?
    (respond_to?(:instance_id) ? instance_id : :default).to_s
  else
    require 'digest'
    urls = Array(config_base_url).map { |u| strip_scheme(u).downcase.chomp('/') }.sort
    Digest::SHA256.hexdigest(urls.join('|'))[0, 12]
  end
end

#cache_local_instance? ⇒ Boolean

── Cache helpers with local/shared tier selection ────────────────

Returns:

  • (Boolean) 


367
368
369
370
371
372
 
# File 'lib/legion/extensions/llm/provider.rb', line 367

def cache_local_instance?
  Array(config_base_url).any? do |url|
    host = url.to_s.downcase
    host.include?('localhost') || host.include?('127.0.0.1') || host.include?('::1')
  end
end

#capabilities ⇒ Object 



57
58
59
 
# File 'lib/legion/extensions/llm/provider.rb', line 57

def capabilities
  self.class.capabilities
end

#chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object

rubocop:disable Metrics/ParameterLists



66
67
68
69
 
# File 'lib/legion/extensions/llm/provider.rb', line 66

def chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil,
         tool_prefs: nil)
  complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:)
end

#complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/legion/extensions/llm/provider.rb', line 76

def complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, thinking: nil,
             tool_prefs: nil, &)
  normalized_temperature = maybe_normalize_temperature(temperature, model)
  log_provider_request(
    messages: messages,
    tools: tools,
    temperature: temperature,
    normalized_temperature: normalized_temperature,
    model: model,
    params: params,
    headers: headers,
    schema: schema,
    thinking: thinking,
    tool_prefs: tool_prefs,
    streaming: block_given?
  )

  payload = Utils.deep_merge(
    render_payload(
      messages,
      tools: tools,
      tool_prefs: tool_prefs,
      temperature: normalized_temperature,
      model: model,
      stream: block_given?,
      schema: schema,
      thinking: thinking
    ),
    params
  )

  if block_given?
    stream_response @connection, payload, headers, &
  else
    sync_response @connection, payload, headers
  end
end

#config_base_url ⇒ Object 



325
326
327
 
# File 'lib/legion/extensions/llm/provider.rb', line 325

def config_base_url
  respond_to?(:settings) ? settings[:base_url] : nil
end

#configuration_requirements ⇒ Object 



61
62
63
 
# File 'lib/legion/extensions/llm/provider.rb', line 61

def configuration_requirements
  self.class.configuration_requirements
end

#configured? ⇒ Boolean

Returns:

  • (Boolean) 


203
204
205
 
# File 'lib/legion/extensions/llm/provider.rb', line 203

def configured?
  configuration_requirements.all? { |req| @config.send(req) }
end

#count_tokens(messages:, model:, params: {}) ⇒ Object 



188
189
190
191
192
193
194
 
# File 'lib/legion/extensions/llm/provider.rb', line 188

def count_tokens(messages:, model:, params: {})
  _ = [model, params]
  Array(messages).sum do |message|
    content = message.respond_to?(:content) ? message.content : message[:content] || message['content']
    estimate_text_tokens(content)
  end
end

#discover_offerings(live: false, **filters) ⇒ Object 



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/legion/extensions/llm/provider.rb', line 121

def discover_offerings(live: false, **filters)
  return filter_cached_offerings(Array(@cached_offerings), filters) unless live

  provider_health = health(live:)
  @cached_offerings = Array(list_models(live:, **filters)).filter_map do |model|
    next unless model_matches_filters?(model, filters)
    next unless model_allowed?(model.id)

    offering_from_model(model, health: provider_health)
  end
  @cached_offerings
rescue Faraday::ConnectionFailed => e
  log.warn("[#{slug}] instance=#{provider_instance_id} unreachable: #{e.message}")
  []
end

#embed(text:, model:, dimensions: nil, params: {}, headers: {}) ⇒ Object 



163
164
165
166
167
168
169
 
# File 'lib/legion/extensions/llm/provider.rb', line 163

def embed(text:, model:, dimensions: nil, params: {}, headers: {})
  payload = Utils.deep_merge(render_embedding_payload(text, model:, dimensions:), params)
  response = @connection.post(embedding_url(model:), payload) do |req|
    req.headers = headers.merge(req.headers) unless headers.empty?
  end
  parse_embedding_response(response, model:, text:)
end

#endpoint_manifest ⇒ Object 



241
242
243
244
245
246
247
248
249
250
 
# File 'lib/legion/extensions/llm/provider.rb', line 241

def endpoint_manifest
  endpoint_methods.each_with_object({}) do |(key, method_name), result|
    next unless respond_to?(method_name)

    value = public_send(method_name)
    result[key] = value unless value.nil?
  rescue ArgumentError, NotImplementedError
    next
  end
end

#fetch_model_detail(_model_name) ⇒ Object

Override in subclasses to make a live API call for model detail. Must return a Hash with symbol keys (e.g. { context_window: 128000 }).



399
400
401
 
# File 'lib/legion/extensions/llm/provider.rb', line 399

def fetch_model_detail(_model_name)
  nil
end

#find_reachable_url(urls) ⇒ Object 



337
338
339
340
341
342
343
 
# File 'lib/legion/extensions/llm/provider.rb', line 337

def find_reachable_url(urls)
  urls.each do |url|
    full = normalize_url(url)
    return full if url_reachable?(full)
  end
  nil
end

#format_messages(messages) ⇒ Object 



274
275
276
277
278
279
280
281
 
# File 'lib/legion/extensions/llm/provider.rb', line 274

def format_messages(messages)
  messages.map do |msg|
    {
      role: msg.role.to_s,
      content: msg.content
    }
  end
end

#format_tool_calls(_tool_calls) ⇒ Object 



283
284
285
 
# File 'lib/legion/extensions/llm/provider.rb', line 283

def format_tool_calls(_tool_calls)
  nil
end

#headers ⇒ Object 



45
46
47
 
# File 'lib/legion/extensions/llm/provider.rb', line 45

def headers
  {}
end

#health(live: false) ⇒ Object 



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/legion/extensions/llm/provider.rb', line 137

def health(live: false)
  readiness_data = readiness(live:)
  raw_health = readiness_data[:health] || readiness_data['health'] || {}
  status = health_status(readiness_data, raw_health)
  {
    provider: slug.to_sym,
    instance_id: provider_instance_id,
    status:,
    ready: readiness_data[:ready] == true || readiness_data['ready'] == true,
    circuit_state: status == 'healthy' ? 'closed' : 'open',
    latency_ms: raw_health[:latency_ms] || raw_health['latency_ms'],
    raw: raw_health
  }.compact
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.health')
  {
    provider: slug.to_sym,
    instance_id: provider_instance_id,
    status: 'unhealthy',
    ready: false,
    circuit_state: 'open',
    error: e.class.name,
    message: e.message
  }
end

#image(prompt:, model:, size:, with: nil, mask: nil, params: {}) ⇒ Object

rubocop:disable Metrics/ParameterLists



184
185
186
 
# File 'lib/legion/extensions/llm/provider.rb', line 184

def image(prompt:, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Metrics/ParameterLists
  paint(prompt, model:, size:, with:, mask:, params:)
end

#list_models(live: false, **filters) ⇒ Object

rubocop:enable Metrics/ParameterLists



115
116
117
118
119
 
# File 'lib/legion/extensions/llm/provider.rb', line 115

def list_models(live: false, **filters)
  _ = [live, filters]
  response = @connection.get models_url
  parse_list_models_response response, slug, capabilities
end

#local? ⇒ Boolean

Returns:

  • (Boolean) 


207
208
209
 
# File 'lib/legion/extensions/llm/provider.rb', line 207

def local?
  self.class.local?
end

#model_allowed?(model_name) ⇒ Boolean

Returns:

  • (Boolean) 


305
306
307
308
309
310
311
312
313
314
 
# File 'lib/legion/extensions/llm/provider.rb', line 305

def model_allowed?(model_name)
  name = model_name.to_s.downcase
  wl = model_whitelist
  bl = model_blacklist

  return false if wl.any? && wl.none? { |p| name.include?(p) }
  return false if bl.any? && bl.any? { |p| name.include?(p) }

  true
end

#model_blacklist ⇒ Object 



299
300
301
302
303
 
# File 'lib/legion/extensions/llm/provider.rb', line 299

def model_blacklist
  bl = config.model_blacklist if config.respond_to?(:model_blacklist)
  bl ||= settings[:model_blacklist] if respond_to?(:settings)
  Array(bl).map { |p| p.to_s.downcase }
end

#model_cache_get(key) ⇒ Object 



374
375
376
377
378
379
380
381
 
# File 'lib/legion/extensions/llm/provider.rb', line 374

def model_cache_get(key)
  return nil unless defined?(Legion::Cache)

  cache_local_instance? ? local_cache_get(key) : cache_get(key)
rescue StandardError => e
  handle_exception(e, level: :debug, handled: true, operation: 'llm.provider.model_cache_get', key:)
  nil
end

#model_detail(model_name) ⇒ Object 



383
384
385
386
387
388
389
390
391
392
393
394
395
 
# File 'lib/legion/extensions/llm/provider.rb', line 383

def model_detail(model_name)
  key = model_detail_cache_key(model_name)
  cached = cache_get(key)
  return cached if cached

  result = fetch_model_detail(model_name)
  cache_set(key, result, ttl: 86_400) if result
  result
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.model_detail',
                      model: model_name)
  nil
end

#model_whitelist ⇒ Object

── Model allow-list / deny-list filtering ────────────────────────



293
294
295
296
297
 
# File 'lib/legion/extensions/llm/provider.rb', line 293

def model_whitelist
  wl = config.model_whitelist if config.respond_to?(:model_whitelist)
  wl ||= settings[:model_whitelist] if respond_to?(:settings)
  Array(wl).map { |p| p.to_s.downcase }
end

#moderate(input, model:) ⇒ Object 



171
172
173
174
175
 
# File 'lib/legion/extensions/llm/provider.rb', line 171

def moderate(input, model:)
  payload = render_moderation_payload(input, model:)
  response = @connection.post moderation_url, payload
  parse_moderation_response(response, model:)
end

#name ⇒ Object 



53
54
55
 
# File 'lib/legion/extensions/llm/provider.rb', line 53

def name
  self.class.name
end

#normalize_url(url) ⇒ Object 



329
330
331
332
333
334
335
 
# File 'lib/legion/extensions/llm/provider.rb', line 329

def normalize_url(url)
  raw = url.to_s.strip
  return raw if raw.match?(%r{^https?://})

  scheme = tls_enabled? ? 'https' : 'http'
  "#{scheme}://#{raw}"
end

#paint(prompt, model:, size:, with: nil, mask: nil, params: {}) ⇒ Object

rubocop:disable Metrics/ParameterLists



177
178
179
180
181
182
 
# File 'lib/legion/extensions/llm/provider.rb', line 177

def paint(prompt, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Metrics/ParameterLists
  validate_paint_inputs!(with:, mask:)
  payload = render_image_payload(prompt, model:, size:, with:, mask:, params:)
  response = @connection.post images_url(with:, mask:), payload
  parse_image_response(response, model:)
end

#parse_error(response) ⇒ Object 



252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
 
# File 'lib/legion/extensions/llm/provider.rb', line 252

def parse_error(response)
  return if response.body.empty?

  body = try_parse_json(response.body)
  case body
  when Hash
    error = body['error']
    return error if error.is_a?(String)

    body.dig('error', 'message')
  when Array
    body.map do |part|
      error = part['error']
      error.is_a?(String) ? error : part.dig('error', 'message')
    end.join('. ')
  when String
    body[/"message"\s*:\s*"([^"]{1,500})/, 1] || body
  else
    body
  end
end

#parse_tool_calls(_tool_calls) ⇒ Object 



287
288
289
 
# File 'lib/legion/extensions/llm/provider.rb', line 287

def parse_tool_calls(_tool_calls)
  nil
end

#provider_instance_id ⇒ Object 



413
414
415
416
417
 
# File 'lib/legion/extensions/llm/provider.rb', line 413

def provider_instance_id
  return config.instance_id.to_sym if config.respond_to?(:instance_id) && config.instance_id

  :default
end

#readiness(live: false) ⇒ Object 



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
 
# File 'lib/legion/extensions/llm/provider.rb', line 219

def readiness(live: false)
  metadata = {
    provider: slug.to_sym,
    name: name,
    configured: configured?,
    ready: configured?,
    local: local?,
    remote: remote?,
    api_base: api_base,
    endpoints: endpoint_manifest,
    live: live
  }

  return metadata.merge(health: { checked: false }) unless live && metadata[:endpoints][:health]

  response = @connection.get(metadata[:endpoints][:health])
  metadata.merge(ready: configured? && health_ready?(response.body), health: response.body)
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.readiness')
  metadata.merge(ready: false, health: { error: e.class.name, message: e.message })
end

#remote? ⇒ Boolean

Returns:

  • (Boolean) 


211
212
213
 
# File 'lib/legion/extensions/llm/provider.rb', line 211

def remote?
  self.class.remote?
end

#resolve_base_url ⇒ Object

── Multi-host base_url resolution ────────────────────────────────



318
319
320
321
322
323
 
# File 'lib/legion/extensions/llm/provider.rb', line 318

def resolve_base_url
  urls = Array(config_base_url)
  return nil if urls.empty?

  @resolve_base_url ||= find_reachable_url(urls) || normalize_url(urls.first)
end

#slug ⇒ Object 



49
50
51
 
# File 'lib/legion/extensions/llm/provider.rb', line 49

def slug
  self.class.slug
end

#stream_chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object 



71
72
73
74
 
# File 'lib/legion/extensions/llm/provider.rb', line 71

def stream_chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil,
                thinking: nil, tool_prefs: nil, &)
  complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:, &)
end

#strip_scheme(url) ⇒ Object 



345
346
347
 
# File 'lib/legion/extensions/llm/provider.rb', line 345

def strip_scheme(url)
  url.to_s.sub(%r{^https?://}, '')
end

#tls_enabled? ⇒ Boolean

Returns:

  • (Boolean) 


360
361
362
363
 
# File 'lib/legion/extensions/llm/provider.rb', line 360

def tls_enabled?
  tls = respond_to?(:settings) ? settings[:tls] : nil
  tls.is_a?(Hash) && tls[:enabled] == true
end

#transcribe(audio_file, model:, language:) ⇒ Object 



196
197
198
199
200
201
 
# File 'lib/legion/extensions/llm/provider.rb', line 196

def transcribe(audio_file, model:, language:, **)
  file_part = build_audio_file_part(audio_file)
  payload = render_transcription_payload(file_part, model:, language:, **)
  response = @connection.post transcription_url, payload
  parse_transcription_response(response, model:)
end

#url_reachable?(url) ⇒ Boolean

Returns:

  • (Boolean) 


349
350
351
352
353
354
355
356
357
358
 
# File 'lib/legion/extensions/llm/provider.rb', line 349

def url_reachable?(url)
  require 'uri'
  require 'socket'
  uri = URI.parse(url)
  Socket.tcp(uri.host, uri.port, connect_timeout: 1).close
  true
rescue StandardError => e
  handle_exception(e, level: :debug, handled: true, operation: 'llm.provider.url_reachable', url:)
  false
end