Class: Legion::Extensions::Llm::Provider

Inherits:
Object
  • Object
show all
Includes:
Streaming, Logging::Helper
Defined in:
lib/legion/extensions/llm/provider.rb,
lib/legion/extensions/llm/provider/open_ai_compatible.rb

Overview

Base class for LLM providers.

Defined Under Namespace

Modules: OpenAICompatible

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Streaming

build_on_data_handler, build_stream_callback, build_stream_error_response, error_chunk?, faraday_1?, handle_data, handle_error_chunk, handle_error_event, handle_failed_response, handle_json_error_chunk, handle_parsed_error, handle_sse, handle_stream, json_error_payload?, parse_error_from_json, parse_streaming_error, process_stream_chunk, stream_response

Constructor Details

#initialize(config) ⇒ Provider

Returns a new instance of Provider.



34
35
36
37
38
 
# File 'lib/legion/extensions/llm/provider.rb', line 34

def initialize(config)
  @config = config.is_a?(Hash) ? HashConfig.new(config) : config
  ensure_configured!
  @connection = Connection.new(self, @config)
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



32
33
34
 
# File 'lib/legion/extensions/llm/provider.rb', line 32

def config
  @config
end

#connection ⇒ Object (readonly)

Returns the value of attribute connection.



32
33
34
 
# File 'lib/legion/extensions/llm/provider.rb', line 32

def connection
  @connection
end

Class Method Details

.assume_models_exist? ⇒ Boolean

Returns:

  • (Boolean) 


436
437
438
 
# File 'lib/legion/extensions/llm/provider.rb', line 436

def assume_models_exist?
  false
end

.capabilities ⇒ Object 



416
417
418
 
# File 'lib/legion/extensions/llm/provider.rb', line 416

def capabilities
  nil
end

.configuration_options ⇒ Object 



424
425
426
 
# File 'lib/legion/extensions/llm/provider.rb', line 424

def configuration_options
  []
end

.configuration_requirements ⇒ Object 



420
421
422
 
# File 'lib/legion/extensions/llm/provider.rb', line 420

def configuration_requirements
  []
end

.configured?(config) ⇒ Boolean

Returns:

  • (Boolean) 


444
445
446
 
# File 'lib/legion/extensions/llm/provider.rb', line 444

def configured?(config)
  configuration_requirements.all? { |req| config.send(req) }
end

.local? ⇒ Boolean

Returns:

  • (Boolean) 


428
429
430
 
# File 'lib/legion/extensions/llm/provider.rb', line 428

def local?
  false
end

.name ⇒ Object 



408
409
410
 
# File 'lib/legion/extensions/llm/provider.rb', line 408

def name
  to_s.split('::').last
end

.remote? ⇒ Boolean

Returns:

  • (Boolean) 


432
433
434
 
# File 'lib/legion/extensions/llm/provider.rb', line 432

def remote?
  !local?
end

.resolve_model_id(model_id, config: nil) ⇒ Object

rubocop:disable Lint/UnusedMethodArgument



440
441
442
 
# File 'lib/legion/extensions/llm/provider.rb', line 440

def resolve_model_id(model_id, config: nil) # rubocop:disable Lint/UnusedMethodArgument
  model_id
end

.slug ⇒ Object 



412
413
414
 
# File 'lib/legion/extensions/llm/provider.rb', line 412

def slug
  name.downcase
end

Instance Method Details

#api_base ⇒ Object

Raises:

  • (NotImplementedError) 


40
41
42
 
# File 'lib/legion/extensions/llm/provider.rb', line 40

def api_base
  raise NotImplementedError
end

#assume_models_exist? ⇒ Boolean

Returns:

  • (Boolean) 


210
211
212
 
# File 'lib/legion/extensions/llm/provider.rb', line 210

def assume_models_exist?
  self.class.assume_models_exist?
end

#cache_instance_key ⇒ Object 



391
392
393
394
395
396
397
398
399
 
# File 'lib/legion/extensions/llm/provider.rb', line 391

def cache_instance_key
  if cache_local_instance?
    (respond_to?(:instance_id) ? instance_id : :default).to_s
  else
    require 'digest'
    urls = Array(config_base_url).map { |u| strip_scheme(u).downcase.chomp('/') }.sort
    Digest::SHA256.hexdigest(urls.join('|'))[0, 12]
  end
end

#cache_local_instance? ⇒ Boolean

── Cache helpers with local/shared tier selection ────────────────

Returns:

  • (Boolean) 


358
359
360
361
362
363
 
# File 'lib/legion/extensions/llm/provider.rb', line 358

def cache_local_instance?
  Array(config_base_url).any? do |url|
    host = url.to_s.downcase
    host.include?('localhost') || host.include?('127.0.0.1') || host.include?('::1')
  end
end

#capabilities ⇒ Object 



56
57
58
 
# File 'lib/legion/extensions/llm/provider.rb', line 56

def capabilities
  self.class.capabilities
end

#chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object

rubocop:disable Metrics/ParameterLists



65
66
67
68
 
# File 'lib/legion/extensions/llm/provider.rb', line 65

def chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil,
         tool_prefs: nil)
  complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:)
end

#complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/legion/extensions/llm/provider.rb', line 75

def complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, thinking: nil,
             tool_prefs: nil, &)
  normalized_temperature = maybe_normalize_temperature(temperature, model)
  log_provider_request(
    messages: messages,
    tools: tools,
    temperature: temperature,
    normalized_temperature: normalized_temperature,
    model: model,
    params: params,
    headers: headers,
    schema: schema,
    thinking: thinking,
    tool_prefs: tool_prefs,
    streaming: block_given?
  )

  payload = Utils.deep_merge(
    render_payload(
      messages,
      tools: tools,
      tool_prefs: tool_prefs,
      temperature: normalized_temperature,
      model: model,
      stream: block_given?,
      schema: schema,
      thinking: thinking
    ),
    params
  )

  if block_given?
    stream_response @connection, payload, headers, &
  else
    sync_response @connection, payload, headers
  end
end

#config_base_url ⇒ Object 



316
317
318
 
# File 'lib/legion/extensions/llm/provider.rb', line 316

def config_base_url
  respond_to?(:settings) ? settings[:base_url] : nil
end

#configuration_requirements ⇒ Object 



60
61
62
 
# File 'lib/legion/extensions/llm/provider.rb', line 60

def configuration_requirements
  self.class.configuration_requirements
end

#configured? ⇒ Boolean

Returns:

  • (Boolean) 


198
199
200
 
# File 'lib/legion/extensions/llm/provider.rb', line 198

def configured?
  configuration_requirements.all? { |req| @config.send(req) }
end

#count_tokens(messages:, model:, params: {}) ⇒ Object 



183
184
185
186
187
188
189
 
# File 'lib/legion/extensions/llm/provider.rb', line 183

def count_tokens(messages:, model:, params: {})
  _ = [model, params]
  Array(messages).sum do |message|
    content = message.respond_to?(:content) ? message.content : message[:content] || message['content']
    estimate_text_tokens(content)
  end
end

#discover_offerings(live: false, **filters) ⇒ Object 



120
121
122
123
124
125
126
127
128
129
130
 
# File 'lib/legion/extensions/llm/provider.rb', line 120

def discover_offerings(live: false, **filters)
  return filter_cached_offerings(Array(@cached_offerings), filters) unless live

  provider_health = health(live:)
  @cached_offerings = Array(list_models(live:, **filters)).filter_map do |model|
    next unless model_matches_filters?(model, filters)

    offering_from_model(model, health: provider_health)
  end
  @cached_offerings
end

#embed(text:, model:, dimensions: nil, params: {}, headers: {}) ⇒ Object 



158
159
160
161
162
163
164
 
# File 'lib/legion/extensions/llm/provider.rb', line 158

def embed(text:, model:, dimensions: nil, params: {}, headers: {})
  payload = Utils.deep_merge(render_embedding_payload(text, model:, dimensions:), params)
  response = @connection.post(embedding_url(model:), payload) do |req|
    req.headers = headers.merge(req.headers) unless headers.empty?
  end
  parse_embedding_response(response, model:, text:)
end

#endpoint_manifest ⇒ Object 



236
237
238
239
240
241
242
243
244
245
 
# File 'lib/legion/extensions/llm/provider.rb', line 236

def endpoint_manifest
  endpoint_methods.each_with_object({}) do |(key, method_name), result|
    next unless respond_to?(method_name)

    value = public_send(method_name)
    result[key] = value unless value.nil?
  rescue ArgumentError, NotImplementedError
    next
  end
end

#find_reachable_url(urls) ⇒ Object 



328
329
330
331
332
333
334
 
# File 'lib/legion/extensions/llm/provider.rb', line 328

def find_reachable_url(urls)
  urls.each do |url|
    full = normalize_url(url)
    return full if url_reachable?(full)
  end
  nil
end

#format_messages(messages) ⇒ Object 



267
268
269
270
271
272
273
274
 
# File 'lib/legion/extensions/llm/provider.rb', line 267

def format_messages(messages)
  messages.map do |msg|
    {
      role: msg.role.to_s,
      content: msg.content
    }
  end
end

#format_tool_calls(_tool_calls) ⇒ Object 



276
277
278
 
# File 'lib/legion/extensions/llm/provider.rb', line 276

def format_tool_calls(_tool_calls)
  nil
end

#headers ⇒ Object 



44
45
46
 
# File 'lib/legion/extensions/llm/provider.rb', line 44

def headers
  {}
end

#health(live: false) ⇒ Object 



132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/legion/extensions/llm/provider.rb', line 132

def health(live: false)
  readiness_data = readiness(live:)
  raw_health = readiness_data[:health] || readiness_data['health'] || {}
  status = health_status(readiness_data, raw_health)
  {
    provider: slug.to_sym,
    instance_id: provider_instance_id,
    status:,
    ready: readiness_data[:ready] == true || readiness_data['ready'] == true,
    circuit_state: status == 'healthy' ? 'closed' : 'open',
    latency_ms: raw_health[:latency_ms] || raw_health['latency_ms'],
    raw: raw_health
  }.compact
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.health')
  {
    provider: slug.to_sym,
    instance_id: provider_instance_id,
    status: 'unhealthy',
    ready: false,
    circuit_state: 'open',
    error: e.class.name,
    message: e.message
  }
end

#image(prompt:, model:, size:, with: nil, mask: nil, params: {}) ⇒ Object

rubocop:disable Metrics/ParameterLists



179
180
181
 
# File 'lib/legion/extensions/llm/provider.rb', line 179

def image(prompt:, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Metrics/ParameterLists
  paint(prompt, model:, size:, with:, mask:, params:)
end

#list_models(live: false, **filters) ⇒ Object

rubocop:enable Metrics/ParameterLists



114
115
116
117
118
 
# File 'lib/legion/extensions/llm/provider.rb', line 114

def list_models(live: false, **filters)
  _ = [live, filters]
  response = @connection.get models_url
  parse_list_models_response response, slug, capabilities
end

#local? ⇒ Boolean

Returns:

  • (Boolean) 


202
203
204
 
# File 'lib/legion/extensions/llm/provider.rb', line 202

def local?
  self.class.local?
end

#model_allowed?(model_name) ⇒ Boolean

Returns:

  • (Boolean) 


296
297
298
299
300
301
302
303
304
305
 
# File 'lib/legion/extensions/llm/provider.rb', line 296

def model_allowed?(model_name)
  name = model_name.to_s.downcase
  wl = model_whitelist
  bl = model_blacklist

  return false if wl.any? && wl.none? { |p| name.include?(p) }
  return false if bl.any? && bl.any? { |p| name.include?(p) }

  true
end

#model_blacklist ⇒ Object 



291
292
293
294
 
# File 'lib/legion/extensions/llm/provider.rb', line 291

def model_blacklist
  bl = settings[:model_blacklist] if respond_to?(:settings)
  Array(bl).map { |p| p.to_s.downcase }
end

#model_cache_fetch(key, ttl:) ⇒ Object 



382
383
384
385
386
387
388
389
 
# File 'lib/legion/extensions/llm/provider.rb', line 382

def model_cache_fetch(key, ttl:, &)
  return yield unless defined?(Legion::Cache)

  cache_local_instance? ? local_cache_fetch(key, ttl: ttl, &) : cache_fetch(key, ttl: ttl, &)
rescue StandardError => e
  handle_exception(e, level: :debug, handled: true, operation: 'llm.provider.model_cache_fetch', key:)
  yield
end

#model_cache_get(key) ⇒ Object 



365
366
367
368
369
370
371
372
 
# File 'lib/legion/extensions/llm/provider.rb', line 365

def model_cache_get(key)
  return nil unless defined?(Legion::Cache)

  cache_local_instance? ? local_cache_get(key) : cache_get(key)
rescue StandardError => e
  handle_exception(e, level: :debug, handled: true, operation: 'llm.provider.model_cache_get', key:)
  nil
end

#model_cache_set(key, value, ttl:) ⇒ Object 



374
375
376
377
378
379
380
 
# File 'lib/legion/extensions/llm/provider.rb', line 374

def model_cache_set(key, value, ttl:)
  return unless defined?(Legion::Cache)

  cache_local_instance? ? local_cache_set(key, value, ttl: ttl) : cache_set(key, value, ttl: ttl)
rescue StandardError => e
  handle_exception(e, level: :debug, handled: true, operation: 'lex.provider.model_cache_set')
end

#model_whitelist ⇒ Object

── Model allow-list / deny-list filtering ────────────────────────



286
287
288
289
 
# File 'lib/legion/extensions/llm/provider.rb', line 286

def model_whitelist
  wl = settings[:model_whitelist] if respond_to?(:settings)
  Array(wl).map { |p| p.to_s.downcase }
end

#moderate(input, model:) ⇒ Object 



166
167
168
169
170
 
# File 'lib/legion/extensions/llm/provider.rb', line 166

def moderate(input, model:)
  payload = render_moderation_payload(input, model:)
  response = @connection.post moderation_url, payload
  parse_moderation_response(response, model:)
end

#name ⇒ Object 



52
53
54
 
# File 'lib/legion/extensions/llm/provider.rb', line 52

def name
  self.class.name
end

#normalize_url(url) ⇒ Object 



320
321
322
323
324
325
326
 
# File 'lib/legion/extensions/llm/provider.rb', line 320

def normalize_url(url)
  raw = url.to_s.strip
  return raw if raw.match?(%r{^https?://})

  scheme = tls_enabled? ? 'https' : 'http'
  "#{scheme}://#{raw}"
end

#paint(prompt, model:, size:, with: nil, mask: nil, params: {}) ⇒ Object

rubocop:disable Metrics/ParameterLists



172
173
174
175
176
177
 
# File 'lib/legion/extensions/llm/provider.rb', line 172

def paint(prompt, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Metrics/ParameterLists
  validate_paint_inputs!(with:, mask:)
  payload = render_image_payload(prompt, model:, size:, with:, mask:, params:)
  response = @connection.post images_url(with:, mask:), payload
  parse_image_response(response, model:)
end

#parse_error(response) ⇒ Object 



247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
 
# File 'lib/legion/extensions/llm/provider.rb', line 247

def parse_error(response)
  return if response.body.empty?

  body = try_parse_json(response.body)
  case body
  when Hash
    error = body['error']
    return error if error.is_a?(String)

    body.dig('error', 'message')
  when Array
    body.map do |part|
      error = part['error']
      error.is_a?(String) ? error : part.dig('error', 'message')
    end.join('. ')
  else
    body
  end
end

#parse_tool_calls(_tool_calls) ⇒ Object 



280
281
282
 
# File 'lib/legion/extensions/llm/provider.rb', line 280

def parse_tool_calls(_tool_calls)
  nil
end

#provider_instance_id ⇒ Object 



401
402
403
404
405
 
# File 'lib/legion/extensions/llm/provider.rb', line 401

def provider_instance_id
  return config.instance_id.to_sym if config.respond_to?(:instance_id) && config.instance_id

  :default
end

#readiness(live: false) ⇒ Object 



214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/legion/extensions/llm/provider.rb', line 214

def readiness(live: false)
  metadata = {
    provider: slug.to_sym,
    name: name,
    configured: configured?,
    ready: configured?,
    local: local?,
    remote: remote?,
    api_base: api_base,
    endpoints: endpoint_manifest,
    live: live
  }

  return metadata.merge(health: { checked: false }) unless live && metadata[:endpoints][:health]

  response = @connection.get(metadata[:endpoints][:health])
  metadata.merge(ready: configured? && health_ready?(response.body), health: response.body)
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.readiness')
  metadata.merge(ready: false, health: { error: e.class.name, message: e.message })
end

#remote? ⇒ Boolean

Returns:

  • (Boolean) 


206
207
208
 
# File 'lib/legion/extensions/llm/provider.rb', line 206

def remote?
  self.class.remote?
end

#resolve_base_url ⇒ Object

── Multi-host base_url resolution ────────────────────────────────



309
310
311
312
313
314
 
# File 'lib/legion/extensions/llm/provider.rb', line 309

def resolve_base_url
  urls = Array(config_base_url)
  return nil if urls.empty?

  @resolve_base_url ||= find_reachable_url(urls) || normalize_url(urls.first)
end

#slug ⇒ Object 



48
49
50
 
# File 'lib/legion/extensions/llm/provider.rb', line 48

def slug
  self.class.slug
end

#stream_chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object 



70
71
72
73
 
# File 'lib/legion/extensions/llm/provider.rb', line 70

def stream_chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil,
                thinking: nil, tool_prefs: nil, &)
  complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:, &)
end

#strip_scheme(url) ⇒ Object 



336
337
338
 
# File 'lib/legion/extensions/llm/provider.rb', line 336

def strip_scheme(url)
  url.to_s.sub(%r{^https?://}, '')
end

#tls_enabled? ⇒ Boolean

Returns:

  • (Boolean) 


351
352
353
354
 
# File 'lib/legion/extensions/llm/provider.rb', line 351

def tls_enabled?
  tls = respond_to?(:settings) ? settings[:tls] : nil
  tls.is_a?(Hash) && tls[:enabled] == true
end

#transcribe(audio_file, model:, language:) ⇒ Object 



191
192
193
194
195
196
 
# File 'lib/legion/extensions/llm/provider.rb', line 191

def transcribe(audio_file, model:, language:, **)
  file_part = build_audio_file_part(audio_file)
  payload = render_transcription_payload(file_part, model:, language:, **)
  response = @connection.post transcription_url, payload
  parse_transcription_response(response, model:)
end

#url_reachable?(url) ⇒ Boolean

Returns:

  • (Boolean) 


340
341
342
343
344
345
346
347
348
349
 
# File 'lib/legion/extensions/llm/provider.rb', line 340

def url_reachable?(url)
  require 'uri'
  require 'socket'
  uri = URI.parse(url)
  Socket.tcp(uri.host, uri.port, connect_timeout: 1).close
  true
rescue StandardError => e
  handle_exception(e, level: :debug, handled: true, operation: 'llm.provider.url_reachable', url:)
  false
end