Module: Legion::LLM::Cache

Extended by:
Legion::Logging::Helper
Defined in:
lib/legion/llm/cache.rb

Constant Summary collapse

DEFAULT_TTL = 
300

Class Method Summary collapse

Class Method Details

.enabled? ⇒ Boolean

Returns true if response caching is enabled in settings and Legion::Cache is loaded.

Returns:

  • (Boolean) 


57
58
59
60
61
62
 
# File 'lib/legion/llm/cache.rb', line 57

def enabled?
  return false unless available?

  settings = llm_settings
  settings.dig(:prompt_caching, :response_cache, :enabled) != false
end

.get(cache_key) ⇒ Object

Returns the cached response hash, or nil on miss / cache unavailable.



29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/legion/llm/cache.rb', line 29

def get(cache_key)
  return nil unless available?

  raw = Legion::Cache.get(cache_key)
  if raw.nil?
    log.debug("LLM cache miss key=#{cache_key}")
    return nil
  end

  ::JSON.parse(raw, symbolize_names: true)
rescue StandardError => e
  handle_exception(e, level: :warn)
  nil
end

.key(model:, provider:, messages:, temperature: nil, tools: nil, schema: nil) ⇒ Object

Generates a deterministic SHA256 cache key from request parameters.



16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/legion/llm/cache.rb', line 16

def key(model:, provider:, messages:, temperature: nil, tools: nil, schema: nil)
  payload = ::JSON.dump({
                          model:       model.to_s,
                          provider:    provider.to_s,
                          messages:    messages,
                          temperature: temperature,
                          tools:       tools,
                          schema:      schema
                        })
  Digest::SHA256.hexdigest(payload)
end

.set(cache_key, response, ttl: DEFAULT_TTL) ⇒ Object

Stores a response in the cache with the given TTL.



45
46
47
48
49
50
51
52
53
54
 
# File 'lib/legion/llm/cache.rb', line 45

def set(cache_key, response, ttl: DEFAULT_TTL)
  return false unless available?

  Legion::Cache.set(cache_key, ::JSON.dump(response), ttl)
  log.debug("LLM cache write key=#{cache_key} ttl=#{ttl}")
  true
rescue StandardError => e
  handle_exception(e, level: :warn)
  false
end