Module: Legion::LLM::Cache

Extended by:
Cache::Helper, Legion::Logging::Helper
Defined in:
lib/legion/llm/cache.rb,
lib/legion/llm/cache/response.rb

Defined Under Namespace

Modules: Response

Constant Summary collapse

DEFAULT_TTL = 
300
RESPONSE_CACHE_SCHEMA_VERSION = 
2

Class Method Summary collapse

Class Method Details

.cache_namespace ⇒ Object 



18
 
# File 'lib/legion/llm/cache.rb', line 18

def cache_namespace = ''

.enabled? ⇒ Boolean

Returns true if response caching is enabled in settings and Legion::Cache is loaded.

Returns:

  • (Boolean) 


63
64
65
66
67
 
# File 'lib/legion/llm/cache.rb', line 63

def enabled?
  return false unless available?

  Legion::Settings.dig(:llm, :prompt_caching, :response_cache, :enabled) != false
end

.get(cache_key) ⇒ Object

Returns the cached response hash, or nil on miss / cache unavailable.



35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/legion/llm/cache.rb', line 35

def get(cache_key)
  return nil unless available?

  raw = cache_backend_get(cache_key)
  if raw.nil?
    log.debug("LLM cache miss key=#{cache_key}")
    return nil
  end

  Legion::JSON.load(raw)
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.cache.get', key: cache_key)
  nil
end

.key(model:, provider:, messages:, temperature: nil, tools: nil, schema: nil) ⇒ Object

Generates a deterministic SHA256 cache key from request parameters.



21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/legion/llm/cache.rb', line 21

def key(model:, provider:, messages:, temperature: nil, tools: nil, schema: nil)
  payload = Legion::JSON.dump({
                                schema_version: RESPONSE_CACHE_SCHEMA_VERSION,
                                model:          model.to_s,
                                provider:       provider.to_s,
                                messages:       messages,
                                temperature:    temperature,
                                tools:          tools,
                                schema:         schema
                              })
  Digest::SHA256.hexdigest(payload)
end

.set(cache_key, response, ttl: DEFAULT_TTL) ⇒ Object

Stores a response in the cache with the given TTL.



51
52
53
54
55
56
57
58
59
60
 
# File 'lib/legion/llm/cache.rb', line 51

def set(cache_key, response, ttl: DEFAULT_TTL)
  return false unless available?

  cache_backend_set(cache_key, Legion::JSON.dump(response), ttl: ttl)
  log.debug("LLM cache write key=#{cache_key} ttl=#{ttl}")
  true
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.cache.set', key: cache_key)
  false
end