Class: Legion::Extensions::Llm::Provider

Inherits:
Object
  • Object
show all
Includes:
Streaming, Logging::Helper
Defined in:
lib/legion/extensions/llm/provider.rb,
lib/legion/extensions/llm/provider/open_ai_compatible.rb

Overview

Base class for LLM providers.

Defined Under Namespace

Modules: OpenAICompatible

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Streaming

build_on_data_handler, build_stream_callback, build_stream_error_response, error_chunk?, faraday_1?, handle_data, handle_error_chunk, handle_error_event, handle_failed_response, handle_json_error_chunk, handle_parsed_error, handle_sse, handle_stream, json_error_payload?, parse_error_from_json, parse_streaming_error, process_stream_chunk, stream_response

Constructor Details

#initialize(config) ⇒ Provider

Returns a new instance of Provider.



34
35
36
37
38
 
# File 'lib/legion/extensions/llm/provider.rb', line 34

def initialize(config)
  @config = config.is_a?(Hash) ? HashConfig.new(config) : config
  ensure_configured!
  @connection = Connection.new(self, @config)
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



32
33
34
 
# File 'lib/legion/extensions/llm/provider.rb', line 32

def config
  @config
end

#connection ⇒ Object (readonly)

Returns the value of attribute connection.



32
33
34
 
# File 'lib/legion/extensions/llm/provider.rb', line 32

def connection
  @connection
end

Class Method Details

.assume_models_exist? ⇒ Boolean

Returns:

  • (Boolean) 


420
421
422
 
# File 'lib/legion/extensions/llm/provider.rb', line 420

def assume_models_exist?
  false
end

.capabilities ⇒ Object 



400
401
402
 
# File 'lib/legion/extensions/llm/provider.rb', line 400

def capabilities
  nil
end

.configuration_options ⇒ Object 



408
409
410
 
# File 'lib/legion/extensions/llm/provider.rb', line 408

def configuration_options
  []
end

.configuration_requirements ⇒ Object 



404
405
406
 
# File 'lib/legion/extensions/llm/provider.rb', line 404

def configuration_requirements
  []
end

.configured?(config) ⇒ Boolean

Returns:

  • (Boolean) 


428
429
430
 
# File 'lib/legion/extensions/llm/provider.rb', line 428

def configured?(config)
  configuration_requirements.all? { |req| config.send(req) }
end

.local? ⇒ Boolean

Returns:

  • (Boolean) 


412
413
414
 
# File 'lib/legion/extensions/llm/provider.rb', line 412

def local?
  false
end

.name ⇒ Object 



392
393
394
 
# File 'lib/legion/extensions/llm/provider.rb', line 392

def name
  to_s.split('::').last
end

.remote? ⇒ Boolean

Returns:

  • (Boolean) 


416
417
418
 
# File 'lib/legion/extensions/llm/provider.rb', line 416

def remote?
  !local?
end

.resolve_model_id(model_id, config: nil) ⇒ Object

rubocop:disable Lint/UnusedMethodArgument



424
425
426
 
# File 'lib/legion/extensions/llm/provider.rb', line 424

def resolve_model_id(model_id, config: nil) # rubocop:disable Lint/UnusedMethodArgument
  model_id
end

.slug ⇒ Object 



396
397
398
 
# File 'lib/legion/extensions/llm/provider.rb', line 396

def slug
  name.downcase
end

Instance Method Details

#api_base ⇒ Object

Raises:

  • (NotImplementedError) 


40
41
42
 
# File 'lib/legion/extensions/llm/provider.rb', line 40

def api_base
  raise NotImplementedError
end

#assume_models_exist? ⇒ Boolean

Returns:

  • (Boolean) 


197
198
199
 
# File 'lib/legion/extensions/llm/provider.rb', line 197

def assume_models_exist?
  self.class.assume_models_exist?
end

#cache_instance_key ⇒ Object 



375
376
377
378
379
380
381
382
383
 
# File 'lib/legion/extensions/llm/provider.rb', line 375

def cache_instance_key
  if cache_local_instance?
    (respond_to?(:instance_id) ? instance_id : :default).to_s
  else
    require 'digest'
    urls = Array(config_base_url).map { |u| strip_scheme(u).downcase.chomp('/') }.sort
    Digest::SHA256.hexdigest(urls.join('|'))[0, 12]
  end
end

#cache_local_instance? ⇒ Boolean

── Cache helpers with local/shared tier selection ────────────────

Returns:

  • (Boolean) 


344
345
346
347
348
349
 
# File 'lib/legion/extensions/llm/provider.rb', line 344

def cache_local_instance?
  Array(config_base_url).any? do |url|
    host = url.to_s.downcase
    host.include?('localhost') || host.include?('127.0.0.1') || host.include?('::1')
  end
end

#capabilities ⇒ Object 



56
57
58
 
# File 'lib/legion/extensions/llm/provider.rb', line 56

def capabilities
  self.class.capabilities
end

#chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object

rubocop:disable Metrics/ParameterLists



65
66
67
68
 
# File 'lib/legion/extensions/llm/provider.rb', line 65

def chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil,
         tool_prefs: nil)
  complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:)
end

#complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/legion/extensions/llm/provider.rb', line 75

def complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, thinking: nil,
             tool_prefs: nil, &)
  normalized_temperature = maybe_normalize_temperature(temperature, model)

  payload = Utils.deep_merge(
    render_payload(
      messages,
      tools: tools,
      tool_prefs: tool_prefs,
      temperature: normalized_temperature,
      model: model,
      stream: block_given?,
      schema: schema,
      thinking: thinking
    ),
    params
  )

  if block_given?
    stream_response @connection, payload, headers, &
  else
    sync_response @connection, payload, headers
  end
end

#config_base_url ⇒ Object 



303
304
305
 
# File 'lib/legion/extensions/llm/provider.rb', line 303

def config_base_url
  respond_to?(:settings) ? settings[:base_url] : nil
end

#configuration_requirements ⇒ Object 



60
61
62
 
# File 'lib/legion/extensions/llm/provider.rb', line 60

def configuration_requirements
  self.class.configuration_requirements
end

#configured? ⇒ Boolean

Returns:

  • (Boolean) 


185
186
187
 
# File 'lib/legion/extensions/llm/provider.rb', line 185

def configured?
  configuration_requirements.all? { |req| @config.send(req) }
end

#count_tokens(messages:, model:, params: {}) ⇒ Object 



170
171
172
173
174
175
176
 
# File 'lib/legion/extensions/llm/provider.rb', line 170

def count_tokens(messages:, model:, params: {})
  _ = [model, params]
  Array(messages).sum do |message|
    content = message.respond_to?(:content) ? message.content : message[:content] || message['content']
    estimate_text_tokens(content)
  end
end

#discover_offerings(live: false, **filters) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/legion/extensions/llm/provider.rb', line 107

def discover_offerings(live: false, **filters)
  return filter_cached_offerings(Array(@cached_offerings), filters) unless live

  provider_health = health(live:)
  @cached_offerings = Array(list_models(live:, **filters)).filter_map do |model|
    next unless model_matches_filters?(model, filters)

    offering_from_model(model, health: provider_health)
  end
  @cached_offerings
end

#embed(text:, model:, dimensions: nil, params: {}, headers: {}) ⇒ Object 



145
146
147
148
149
150
151
 
# File 'lib/legion/extensions/llm/provider.rb', line 145

def embed(text:, model:, dimensions: nil, params: {}, headers: {})
  payload = Utils.deep_merge(render_embedding_payload(text, model:, dimensions:), params)
  response = @connection.post(embedding_url(model:), payload) do |req|
    req.headers = headers.merge(req.headers) unless headers.empty?
  end
  parse_embedding_response(response, model:, text:)
end

#endpoint_manifest ⇒ Object 



223
224
225
226
227
228
229
230
231
232
 
# File 'lib/legion/extensions/llm/provider.rb', line 223

def endpoint_manifest
  endpoint_methods.each_with_object({}) do |(key, method_name), result|
    next unless respond_to?(method_name)

    value = public_send(method_name)
    result[key] = value unless value.nil?
  rescue ArgumentError, NotImplementedError
    next
  end
end

#find_reachable_url(urls) ⇒ Object 



315
316
317
318
319
320
321
 
# File 'lib/legion/extensions/llm/provider.rb', line 315

def find_reachable_url(urls)
  urls.each do |url|
    full = normalize_url(url)
    return full if url_reachable?(full)
  end
  nil
end

#format_messages(messages) ⇒ Object 



254
255
256
257
258
259
260
261
 
# File 'lib/legion/extensions/llm/provider.rb', line 254

def format_messages(messages)
  messages.map do |msg|
    {
      role: msg.role.to_s,
      content: msg.content
    }
  end
end

#format_tool_calls(_tool_calls) ⇒ Object 



263
264
265
 
# File 'lib/legion/extensions/llm/provider.rb', line 263

def format_tool_calls(_tool_calls)
  nil
end

#headers ⇒ Object 



44
45
46
 
# File 'lib/legion/extensions/llm/provider.rb', line 44

def headers
  {}
end

#health(live: false) ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
 
# File 'lib/legion/extensions/llm/provider.rb', line 119

def health(live: false)
  readiness_data = readiness(live:)
  raw_health = readiness_data[:health] || readiness_data['health'] || {}
  status = health_status(readiness_data, raw_health)
  {
    provider: slug.to_sym,
    instance_id: provider_instance_id,
    status:,
    ready: readiness_data[:ready] == true || readiness_data['ready'] == true,
    circuit_state: status == 'healthy' ? 'closed' : 'open',
    latency_ms: raw_health[:latency_ms] || raw_health['latency_ms'],
    raw: raw_health
  }.compact
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.health')
  {
    provider: slug.to_sym,
    instance_id: provider_instance_id,
    status: 'unhealthy',
    ready: false,
    circuit_state: 'open',
    error: e.class.name,
    message: e.message
  }
end

#image(prompt:, model:, size:, with: nil, mask: nil, params: {}) ⇒ Object

rubocop:disable Metrics/ParameterLists



166
167
168
 
# File 'lib/legion/extensions/llm/provider.rb', line 166

def image(prompt:, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Metrics/ParameterLists
  paint(prompt, model:, size:, with:, mask:, params:)
end

#list_models(live: false, **filters) ⇒ Object

rubocop:enable Metrics/ParameterLists



101
102
103
104
105
 
# File 'lib/legion/extensions/llm/provider.rb', line 101

def list_models(live: false, **filters)
  _ = [live, filters]
  response = @connection.get models_url
  parse_list_models_response response, slug, capabilities
end

#local? ⇒ Boolean

Returns:

  • (Boolean) 


189
190
191
 
# File 'lib/legion/extensions/llm/provider.rb', line 189

def local?
  self.class.local?
end

#model_allowed?(model_name) ⇒ Boolean

Returns:

  • (Boolean) 


283
284
285
286
287
288
289
290
291
292
 
# File 'lib/legion/extensions/llm/provider.rb', line 283

def model_allowed?(model_name)
  name = model_name.to_s.downcase
  wl = model_whitelist
  bl = model_blacklist

  return false if wl.any? && wl.none? { |p| name.include?(p) }
  return false if bl.any? && bl.any? { |p| name.include?(p) }

  true
end

#model_blacklist ⇒ Object 



278
279
280
281
 
# File 'lib/legion/extensions/llm/provider.rb', line 278

def model_blacklist
  bl = settings[:model_blacklist] if respond_to?(:settings)
  Array(bl).map { |p| p.to_s.downcase }
end

#model_cache_fetch(key, ttl:) ⇒ Object 



367
368
369
370
371
372
373
 
# File 'lib/legion/extensions/llm/provider.rb', line 367

def model_cache_fetch(key, ttl:, &)
  return yield unless defined?(Legion::Cache)

  cache_local_instance? ? local_cache_fetch(key, ttl: ttl, &) : cache_fetch(key, ttl: ttl, &)
rescue StandardError
  yield
end

#model_cache_get(key) ⇒ Object 



351
352
353
354
355
356
357
 
# File 'lib/legion/extensions/llm/provider.rb', line 351

def model_cache_get(key)
  return nil unless defined?(Legion::Cache)

  cache_local_instance? ? local_cache_get(key) : cache_get(key)
rescue StandardError
  nil
end

#model_cache_set(key, value, ttl:) ⇒ Object 



359
360
361
362
363
364
365
 
# File 'lib/legion/extensions/llm/provider.rb', line 359

def model_cache_set(key, value, ttl:)
  return unless defined?(Legion::Cache)

  cache_local_instance? ? local_cache_set(key, value, ttl: ttl) : cache_set(key, value, ttl: ttl)
rescue StandardError => e
  handle_exception(e, level: :debug, handled: true, operation: 'lex.provider.model_cache_set')
end

#model_whitelist ⇒ Object

── Model allow-list / deny-list filtering ────────────────────────



273
274
275
276
 
# File 'lib/legion/extensions/llm/provider.rb', line 273

def model_whitelist
  wl = settings[:model_whitelist] if respond_to?(:settings)
  Array(wl).map { |p| p.to_s.downcase }
end

#moderate(input, model:) ⇒ Object 



153
154
155
156
157
 
# File 'lib/legion/extensions/llm/provider.rb', line 153

def moderate(input, model:)
  payload = render_moderation_payload(input, model:)
  response = @connection.post moderation_url, payload
  parse_moderation_response(response, model:)
end

#name ⇒ Object 



52
53
54
 
# File 'lib/legion/extensions/llm/provider.rb', line 52

def name
  self.class.name
end

#normalize_url(url) ⇒ Object 



307
308
309
310
311
312
313
 
# File 'lib/legion/extensions/llm/provider.rb', line 307

def normalize_url(url)
  raw = url.to_s.strip
  return raw if raw.match?(%r{^https?://})

  scheme = tls_enabled? ? 'https' : 'http'
  "#{scheme}://#{raw}"
end

#paint(prompt, model:, size:, with: nil, mask: nil, params: {}) ⇒ Object

rubocop:disable Metrics/ParameterLists



159
160
161
162
163
164
 
# File 'lib/legion/extensions/llm/provider.rb', line 159

def paint(prompt, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Metrics/ParameterLists
  validate_paint_inputs!(with:, mask:)
  payload = render_image_payload(prompt, model:, size:, with:, mask:, params:)
  response = @connection.post images_url(with:, mask:), payload
  parse_image_response(response, model:)
end

#parse_error(response) ⇒ Object 



234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/legion/extensions/llm/provider.rb', line 234

def parse_error(response)
  return if response.body.empty?

  body = try_parse_json(response.body)
  case body
  when Hash
    error = body['error']
    return error if error.is_a?(String)

    body.dig('error', 'message')
  when Array
    body.map do |part|
      error = part['error']
      error.is_a?(String) ? error : part.dig('error', 'message')
    end.join('. ')
  else
    body
  end
end

#parse_tool_calls(_tool_calls) ⇒ Object 



267
268
269
 
# File 'lib/legion/extensions/llm/provider.rb', line 267

def parse_tool_calls(_tool_calls)
  nil
end

#provider_instance_id ⇒ Object 



385
386
387
388
389
 
# File 'lib/legion/extensions/llm/provider.rb', line 385

def provider_instance_id
  return config.instance_id.to_sym if config.respond_to?(:instance_id) && config.instance_id

  :default
end

#readiness(live: false) ⇒ Object 



201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
 
# File 'lib/legion/extensions/llm/provider.rb', line 201

def readiness(live: false)
  metadata = {
    provider: slug.to_sym,
    name: name,
    configured: configured?,
    ready: configured?,
    local: local?,
    remote: remote?,
    api_base: api_base,
    endpoints: endpoint_manifest,
    live: live
  }

  return metadata.merge(health: { checked: false }) unless live && metadata[:endpoints][:health]

  response = @connection.get(metadata[:endpoints][:health])
  metadata.merge(ready: configured? && health_ready?(response.body), health: response.body)
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.readiness')
  metadata.merge(ready: false, health: { error: e.class.name, message: e.message })
end

#remote? ⇒ Boolean

Returns:

  • (Boolean) 


193
194
195
 
# File 'lib/legion/extensions/llm/provider.rb', line 193

def remote?
  self.class.remote?
end

#resolve_base_url ⇒ Object

── Multi-host base_url resolution ────────────────────────────────



296
297
298
299
300
301
 
# File 'lib/legion/extensions/llm/provider.rb', line 296

def resolve_base_url
  urls = Array(config_base_url)
  return nil if urls.empty?

  @resolve_base_url ||= find_reachable_url(urls) || normalize_url(urls.first)
end

#slug ⇒ Object 



48
49
50
 
# File 'lib/legion/extensions/llm/provider.rb', line 48

def slug
  self.class.slug
end

#stream_chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil, tool_prefs: nil) ⇒ Object 



70
71
72
73
 
# File 'lib/legion/extensions/llm/provider.rb', line 70

def stream_chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil,
                thinking: nil, tool_prefs: nil, &)
  complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:, &)
end

#strip_scheme(url) ⇒ Object 



323
324
325
 
# File 'lib/legion/extensions/llm/provider.rb', line 323

def strip_scheme(url)
  url.to_s.sub(%r{^https?://}, '')
end

#tls_enabled? ⇒ Boolean

Returns:

  • (Boolean) 


337
338
339
340
 
# File 'lib/legion/extensions/llm/provider.rb', line 337

def tls_enabled?
  tls = respond_to?(:settings) ? settings[:tls] : nil
  tls.is_a?(Hash) && tls[:enabled] == true
end

#transcribe(audio_file, model:, language:) ⇒ Object 



178
179
180
181
182
183
 
# File 'lib/legion/extensions/llm/provider.rb', line 178

def transcribe(audio_file, model:, language:, **)
  file_part = build_audio_file_part(audio_file)
  payload = render_transcription_payload(file_part, model:, language:, **)
  response = @connection.post transcription_url, payload
  parse_transcription_response(response, model:)
end

#url_reachable?(url) ⇒ Boolean

Returns:

  • (Boolean) 


327
328
329
330
331
332
333
334
335
 
# File 'lib/legion/extensions/llm/provider.rb', line 327

def url_reachable?(url)
  require 'uri'
  require 'socket'
  uri = URI.parse(url)
  Socket.tcp(uri.host, uri.port, connect_timeout: 1).close
  true
rescue StandardError
  false
end