Module: Legion::LLM::ResponseCache

Extended by:
Legion::Logging::Helper
Defined in:
lib/legion/llm/response_cache.rb

Constant Summary collapse

DEFAULT_TTL = 
300
SPOOL_THRESHOLD =

8 MB

8 * 1024 * 1024
SPOOL_DIR = 
File.expand_path('~/.legionio/data/spool/llm_responses').freeze

Class Method Summary collapse

Class Method Details

.cleanup(request_id) ⇒ Object

Removes all cache keys for a request (and any spool file).



92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/legion/llm/response_cache.rb', line 92

def cleanup(request_id)
  raw = Legion::Cache.get(response_key(request_id))
  if raw&.start_with?('spool:')
    path = raw.delete_prefix('spool:')
    FileUtils.rm_f(path)
  end

  Legion::Cache.delete(status_key(request_id))
  Legion::Cache.delete(response_key(request_id))
  Legion::Cache.delete(meta_key(request_id))
  Legion::Cache.delete(error_key(request_id))
end

.complete(request_id, response:, meta:, ttl: DEFAULT_TTL) ⇒ Object

Writes response, meta, and marks status as :done.



24
25
26
27
28
 
# File 'lib/legion/llm/response_cache.rb', line 24

def complete(request_id, response:, meta:, ttl: DEFAULT_TTL)
  write_response(request_id, response, ttl)
  cache_set(meta_key(request_id), ::JSON.dump(meta), ttl)
  cache_set(status_key(request_id), 'done', ttl)
end

.error(request_id) ⇒ Object

Returns { code:, message: } hash, or nil.



62
63
64
65
66
67
 
# File 'lib/legion/llm/response_cache.rb', line 62

def error(request_id)
  raw = Legion::Cache.get(error_key(request_id))
  return nil if raw.nil?

  ::JSON.parse(raw, symbolize_names: true)
end

.fail_request(request_id, code:, message:, ttl: DEFAULT_TTL) ⇒ Object

Writes error details and marks status as :error.



31
32
33
34
35
36
 
# File 'lib/legion/llm/response_cache.rb', line 31

def fail_request(request_id, code:, message:, ttl: DEFAULT_TTL)
  log.warn("ResponseCache fail_request request_id=#{request_id} code=#{code} message=#{message}")
  payload = ::JSON.dump({ code: code, message: message })
  cache_set(error_key(request_id), payload, ttl)
  cache_set(status_key(request_id), 'error', ttl)
end

.init_request(request_id, ttl: DEFAULT_TTL) ⇒ Object

Sets status to :pending for a new request.



19
20
21
 
# File 'lib/legion/llm/response_cache.rb', line 19

def init_request(request_id, ttl: DEFAULT_TTL)
  cache_set(status_key(request_id), 'pending', ttl)
end

.meta(request_id) ⇒ Object

Returns meta hash with symbolized keys, or nil.



54
55
56
57
58
59
 
# File 'lib/legion/llm/response_cache.rb', line 54

def meta(request_id)
  raw = Legion::Cache.get(meta_key(request_id))
  return nil if raw.nil?

  ::JSON.parse(raw, symbolize_names: true)
end

.poll(request_id, timeout: DEFAULT_TTL, interval: 0.1) ⇒ Object

Blocking poll. Returns { status: :done, response:, meta: }, { status: :error, error: }, or { status: :timeout }.



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/legion/llm/response_cache.rb', line 71

def poll(request_id, timeout: DEFAULT_TTL, interval: 0.1)
  deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout

  loop do
    current = status(request_id)
    log.debug("ResponseCache poll request_id=#{request_id} status=#{current}")

    case current
    when :done
      return { status: :done, response: response(request_id), meta: meta(request_id) }
    when :error
      return { status: :error, error: error(request_id) }
    end

    return { status: :timeout } if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) >= deadline

    sleep interval
  end
end

.response(request_id) ⇒ Object

Returns the response string (handles spool overflow transparently).



45
46
47
48
49
50
51
 
# File 'lib/legion/llm/response_cache.rb', line 45

def response(request_id)
  raw = Legion::Cache.get(response_key(request_id))
  return nil if raw.nil?
  return File.read(raw.delete_prefix('spool:')) if raw.start_with?('spool:')

  raw
end

.status(request_id) ⇒ Object

Returns :pending, :done, :error, or nil.



39
40
41
42
 
# File 'lib/legion/llm/response_cache.rb', line 39

def status(request_id)
  raw = Legion::Cache.get(status_key(request_id))
  raw&.to_sym
end