Module: Legion::LLM::Cache

Extended by:
Cache::Helper, Legion::Logging::Helper
Defined in:
lib/legion/llm/cache.rb,
lib/legion/llm/cache/response.rb

Defined Under Namespace

Modules: Response

Constant Summary collapse

DEFAULT_TTL = 
300
RESPONSE_CACHE_SCHEMA_VERSION = 
2

Class Method Summary collapse

Class Method Details

.cache_namespace ⇒ Object 



18
 
# File 'lib/legion/llm/cache.rb', line 18

def cache_namespace = ''

.enabled? ⇒ Boolean

Returns true if response caching is enabled in settings and Legion::Cache is loaded.

Returns:

  • (Boolean) 


92
93
94
95
96
 
# File 'lib/legion/llm/cache.rb', line 92

def enabled?
  return false unless available?

  Legion::Settings.dig(:llm, :prompt_caching, :response_cache, :enabled) != false
end

.get(cache_key) ⇒ Object

Returns the cached response hash, or nil on miss / cache unavailable.



64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/legion/llm/cache.rb', line 64

def get(cache_key)
  return nil unless available?

  raw = cache_backend_get(cache_key)
  if raw.nil?
    log.debug("LLM cache miss key=#{cache_key}")
    return nil
  end

  Legion::JSON.load(raw)
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.cache.get', key: cache_key)
  nil
end

.key(model:, provider:, messages:, temperature: nil, tools: nil, schema: nil) ⇒ Object

Generates a deterministic SHA256 cache key from request parameters.



21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/legion/llm/cache.rb', line 21

def key(model:, provider:, messages:, temperature: nil, tools: nil, schema: nil)
  payload = Legion::JSON.dump({
                                schema_version: RESPONSE_CACHE_SCHEMA_VERSION,
                                model:          model.to_s,
                                provider:       provider.to_s,
                                messages:       messages,
                                temperature:    temperature,
                                tools:          tools,
                                schema:         schema
                              })
  Digest::SHA256.hexdigest(payload)
end

.selection_key(provider_family:, model:, revision:, operation:, system:, messages:, tools: nil, tool_choice: nil, thinking: nil, response_format: nil, max_output_tokens: nil, generation: nil) ⇒ Object

SSOT v3 §20.1 response-cache identity. Built from the EXACT selected lane (provider_family + selected model + authoritative immutable model revision, or the exact instance_id when revision is unknown) plus every semantic request input that changes the answer. It deliberately never includes lane_id, offering_id, tier, weight, affinity, routing seed, or the requested/ignored body alias — identical model strings across provider families never share, and cross-instance sharing is allowed only with identical authoritative revision evidence.



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/legion/llm/cache.rb', line 42

def selection_key(provider_family:, model:, revision:, operation:, system:, messages:,
                  tools: nil, tool_choice: nil, thinking: nil, response_format: nil,
                  max_output_tokens: nil, generation: nil)
  payload = Legion::JSON.dump({
                                schema_version:  RESPONSE_CACHE_SCHEMA_VERSION,
                                provider_family: provider_family.to_s,
                                model:           model.to_s,
                                revision:        revision.to_s,
                                operation:       operation.to_s,
                                system:          system,
                                messages:        messages,
                                tools:           tools,
                                tool_choice:     tool_choice,
                                thinking:        thinking,
                                response_format: response_format,
                                max_output:      max_output_tokens,
                                generation:      generation
                              })
  Digest::SHA256.hexdigest(payload)
end

.set(cache_key, response, ttl: DEFAULT_TTL) ⇒ Object

Stores a response in the cache with the given TTL.



80
81
82
83
84
85
86
87
88
89
 
# File 'lib/legion/llm/cache.rb', line 80

def set(cache_key, response, ttl: DEFAULT_TTL)
  return false unless available?

  cache_backend_set(cache_key, Legion::JSON.dump(response), ttl: ttl)
  log.debug("LLM cache write key=#{cache_key} ttl=#{ttl}")
  true
rescue StandardError => e
  handle_exception(e, level: :warn, handled: true, operation: 'llm.cache.set', key: cache_key)
  false
end