Class: Legion::Extensions::Llm::Vllm::Provider

Inherits:
Provider
  • Object
show all
Includes:
Provider::OpenAICompatible, ProviderCanonicalRequestBridge, ProviderDiscoveryHelpers, ProviderLegacyBridge, ProviderManagementMethods, ProviderRenderParseHelpers, Logging::Helper
Defined in:
lib/legion/extensions/llm/vllm/provider.rb

Overview

vLLM provider implementation for the Legion::Extensions::Llm base provider contract.

Defined Under Namespace

Modules: Capabilities

Class Method Summary collapse

Instance Method Summary collapse

Methods included from ProviderManagementMethods

#reset_mm_cache, #reset_prefix_cache, #sleep, #version, #wake_up

Class Method Details

.capabilities ⇒ Object 



464
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 464

def capabilities = Capabilities

.configuration_options ⇒ Object 



462
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 462

def configuration_options = %i[vllm_api_base vllm_api_key]

.configuration_requirements ⇒ Object 



463
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 463

def configuration_requirements = []

.default_tier ⇒ Object 



461
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 461

def default_tier = :direct

.default_transport ⇒ Object 



460
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 460

def default_transport = :http

.local? ⇒ Boolean

Returns:

  • (Boolean) 


459
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 459

def local? = false

.registry_publisher ⇒ Object 



466
467
468
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 466

def registry_publisher
  Vllm.registry_publisher
end

.slug ⇒ Object 



458
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 458

def slug = 'vllm'

Instance Method Details

#api_base ⇒ Object 



501
502
503
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 501

def api_base
  normalize_url(config.vllm_api_base || settings.dig(:instances, :default, :endpoint))
end

#discover_offerings(live: false, **filters) ⇒ Object 



538
539
540
541
542
543
544
545
546
547
548
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 538

def discover_offerings(live: false, **filters)
  return filter_cached_offerings(Array(@cached_offerings), filters) unless live

  provider_health = health(live:)
  @cached_offerings = discover_live_offerings(filters, provider_health, live:)
  log_discover_complete(@cached_offerings)
  @cached_offerings
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'vllm.discover_offerings')
  []
end

#headers ⇒ Object 



505
506
507
508
509
510
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 505

def headers
  hdrs = identity_headers
  token = config.vllm_api_key
  hdrs['Authorization'] = "Bearer #{token}" unless token.nil? || token.to_s.empty?
  hdrs
end

#health(live: false) ⇒ Object 



519
520
521
522
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 519

def health(live: false)
  log.info { "checking health live=#{live} at #{api_base}#{health_url}" }
  super
end

#health_url ⇒ Object 



512
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 512

def health_url = '/health'

#list_models(live: false, **filters) ⇒ Object 



531
532
533
534
535
536
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 531

def list_models(live: false, **filters)
  log.info { "discovering models from #{api_base}#{models_url}" }
  super.tap do |models|
    log.info { "discovered #{models.size} model(s) from vLLM" }
  end
end

#readiness(live: false) ⇒ Object 



524
525
526
527
528
529
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 524

def readiness(live: false)
  log.info { "checking readiness live=#{live} at #{api_base}" }
  super.tap do |metadata|
    self.class.registry_publisher.publish_readiness_async(metadata) if live
  end
end

#reset_mm_cache_url ⇒ Object 



515
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 515

def reset_mm_cache_url = '/reset_mm_cache'

#reset_prefix_cache_url ⇒ Object 



514
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 514

def reset_prefix_cache_url = '/reset_prefix_cache'

#settings ⇒ Object 



493
494
495
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 493

def settings
  Vllm.default_settings
end

#sleep_url ⇒ Object 



516
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 516

def sleep_url = '/sleep'

#stream_usage_supported? ⇒ Boolean

Returns:

  • (Boolean) 


491
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 491

def stream_usage_supported? = true

#translator ⇒ Object 



497
498
499
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 497

def translator
  @translator ||= Translator.new(config: config)
end

#version_url ⇒ Object 



513
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 513

def version_url = '/version'

#wake_up_url ⇒ Object 



517
 
# File 'lib/legion/extensions/llm/vllm/provider.rb', line 517

def wake_up_url = '/wake_up'