Class: Langfuse::PromptCache

Inherits:
Object
  • Object
show all
Includes:
StaleWhileRevalidate
Defined in:
lib/langfuse/prompt_cache.rb

Overview

Simple in-memory cache for prompt data with TTL

Thread-safe cache implementation for storing prompt responses with time-to-live expiration.

rubocop:disable Metrics/ClassLength

Examples:

cache = Langfuse::PromptCache.new(ttl: 60)
cache.set("greeting:1", prompt_data)
cache.get("greeting:1") # => prompt_data

Defined Under Namespace

Classes: CacheEntry

Constant Summary collapse

MAX_NAME_GENERATIONS =

Caps the per-name generation map. Without a cap, long-lived processes that invalidate across many distinct prompts grow it unboundedly; LRU eviction keeps the working set live and lets cold names go.

1024

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from StaleWhileRevalidate

#fetch_with_stale_while_revalidate, #initialize_swr, #refresh_async, #shutdown, #swr_enabled?, #write_with_stale_while_revalidate

Constructor Details

#initialize(ttl: 60, max_size: 1000, stale_ttl: 0, refresh_threads: 5, logger: default_logger) ⇒ PromptCache

Initialize a new cache

Parameters:

  • ttl (Integer) (defaults to: 60) —

    Time-to-live in seconds (default: 60)

  • max_size (Integer) (defaults to: 1000) —

    Maximum cache size (default: 1000)

  • stale_ttl (Integer) (defaults to: 0) —

    Stale TTL for SWR in seconds (default: 0, SWR disabled). Note: :indefinite is normalized to 1000 years by Config before being passed here.

  • refresh_threads (Integer) (defaults to: 5) —

    Number of background refresh threads (default: 5)

  • logger (Logger, nil) (defaults to: default_logger) —

    Logger instance for error reporting (default: nil, creates new logger)



83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/langfuse/prompt_cache.rb', line 83

def initialize(ttl: 60, max_size: 1000, stale_ttl: 0, refresh_threads: 5, logger: default_logger)
  @ttl = ttl
  @max_size = max_size
  @stale_ttl = stale_ttl
  @logger = logger
  @cache = {}
  @global_generation = 0
  @name_generations = {}
  @name_generation_counter = 0
  @monitor = Monitor.new
  @locks = {} # Track locks for in-memory locking
  initialize_swr(refresh_threads: refresh_threads) if swr_enabled?
end

Instance Attribute Details

#logger ⇒ Logger (readonly)

Returns Logger instance for error reporting.

Returns:

  • (Logger) —

    Logger instance for error reporting



73
74
75
 
# File 'lib/langfuse/prompt_cache.rb', line 73

def logger
  @logger
end

#max_size ⇒ Integer (readonly)

Returns Maximum number of cache entries.

Returns:

  • (Integer) —

    Maximum number of cache entries



67
68
69
 
# File 'lib/langfuse/prompt_cache.rb', line 67

def max_size
  @max_size
end

#stale_ttl ⇒ Integer (readonly)

Returns Stale TTL for SWR in seconds.

Returns:

  • (Integer) —

    Stale TTL for SWR in seconds



70
71
72
 
# File 'lib/langfuse/prompt_cache.rb', line 70

def stale_ttl
  @stale_ttl
end

#ttl ⇒ Integer (readonly)

Returns Time-to-live in seconds.

Returns:

  • (Integer) —

    Time-to-live in seconds



64
65
66
 
# File 'lib/langfuse/prompt_cache.rb', line 64

def ttl
  @ttl
end

Class Method Details

.build_key(name, version: nil, label: nil) ⇒ String

Build a cache key from prompt name and options

Parameters:

  • name (String) —

    Prompt name

  • version (Integer, nil) (defaults to: nil) —

    Optional version

  • label (String, nil) (defaults to: nil) —

    Optional label

Returns:

  • (String) —

    Cache key



262
263
264
265
266
267
268
 
# File 'lib/langfuse/prompt_cache.rb', line 262

def self.build_key(name, version: nil, label: nil)
  key = name.to_s
  key += ":v#{version}" if version
  key += ":#{label}" if label
  key += ":production" unless version || label
  key
end

.storage_key(logical_key, name:, global_generation:, name_generation:) ⇒ String

Build a generated storage key from generation metadata.

Parameters:

  • logical_key (String) —

    Stable logical cache identity

  • name (String) —

    Prompt name

  • global_generation (Integer) —

    Global cache generation

  • name_generation (Integer) —

    Prompt-name cache generation

Returns:

  • (String) —

    Generated storage key



277
278
279
280
 
# File 'lib/langfuse/prompt_cache.rb', line 277

def self.storage_key(logical_key, name:, global_generation:, name_generation:)
  encoded_name = Base64.urlsafe_encode64(name.to_s, padding: false)
  "g#{global_generation}:n#{encoded_name}:#{name_generation}:#{logical_key}"
end

Instance Method Details

#cleanup_expired ⇒ Integer

Remove expired entries from cache

Returns:

  • (Integer) —

    Number of entries removed



221
222
223
224
225
226
227
 
# File 'lib/langfuse/prompt_cache.rb', line 221

def cleanup_expired
  @monitor.synchronize do
    initial_size = @cache.size
    @cache.delete_if { |_key, entry| entry.expired? }
    initial_size - @cache.size
  end
end

#clear ⇒ void

This method returns an undefined value.

Clear the entire cache



153
154
155
156
157
 
# File 'lib/langfuse/prompt_cache.rb', line 153

def clear
  @monitor.synchronize do
    @cache.clear
  end
end

#clear_logically ⇒ Integer

Logically invalidate every generated storage key.

Returns:

  • (Integer) —

    New global generation



162
163
164
165
166
 
# File 'lib/langfuse/prompt_cache.rb', line 162

def clear_logically
  @monitor.synchronize do
    @global_generation += 1
  end
end

#delete(key) ⇒ Boolean

Delete one generated storage key.

Parameters:

  • key (String) —

    Generated storage key

Returns:

  • (Boolean) —

    true if an entry was removed



144
145
146
147
148
 
# File 'lib/langfuse/prompt_cache.rb', line 144

def delete(key)
  @monitor.synchronize do
    !@cache.delete(key).nil?
  end
end

#empty? ⇒ Boolean

Check if cache is empty

Returns:

  • (Boolean) 


241
242
243
244
245
 
# File 'lib/langfuse/prompt_cache.rb', line 241

def empty?
  @monitor.synchronize do
    @cache.empty?
  end
end

#entry(key) ⇒ CacheEntry?

Read a raw cache entry, including stale entries.

Parameters:

  • key (String) —

    Cache key

Returns:



115
116
117
118
119
 
# File 'lib/langfuse/prompt_cache.rb', line 115

def entry(key)
  @monitor.synchronize do
    @cache[key]
  end
end

#get(key) ⇒ Object?

Get a value from the cache

Parameters:

  • key (String) —

    Cache key

Returns:

  • (Object, nil) —

    Cached value or nil if not found/expired



101
102
103
104
105
106
107
108
109
 
# File 'lib/langfuse/prompt_cache.rb', line 101

def get(key)
  @monitor.synchronize do
    entry = @cache[key]
    return nil unless entry
    return nil if entry.expired?

    entry.data
  end
end

#invalidate_name(name) ⇒ Integer

Logically invalidate every cache variant for one prompt name.

Generations come from a monotonic global counter, not a per-name counter, so an evicted name re-entering the map can’t reuse a generation value that’s still embedded in a stale @cache entry.

Parameters:

  • name (String) —

    Prompt name

Returns:

  • (Integer) —

    New name generation



176
177
178
179
180
181
182
183
184
 
# File 'lib/langfuse/prompt_cache.rb', line 176

def invalidate_name(name)
  @monitor.synchronize do
    name_str = name.to_s
    @name_generations.delete(name_str)
    @name_generations.shift if @name_generations.size >= MAX_NAME_GENERATIONS
    @name_generation_counter += 1
    @name_generations[name_str] = @name_generation_counter
  end
end

#set(key, value, ttl: nil, stale_ttl: nil) ⇒ Object

Set a value in the cache

Parameters:

  • key (String) —

    Cache key

  • value (Object) —

    Value to cache

Returns:

  • (Object) —

    The cached value



126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/langfuse/prompt_cache.rb', line 126

def set(key, value, ttl: nil, stale_ttl: nil)
  @monitor.synchronize do
    # Evict oldest entry if at max size
    evict_oldest if @cache.size >= max_size
    # TTL math is inlined (not extracted to a helper) to keep this hot path
    # allocation-free apart from the CacheEntry below.
    effective_ttl = ttl.nil? ? self.ttl : ttl
    effective_stale_ttl = stale_ttl.nil? ? self.stale_ttl : stale_ttl
    fresh_until = Time.now + effective_ttl
    @cache[key] = CacheEntry.new(value, fresh_until, fresh_until + effective_stale_ttl)
    value
  end
end

#size ⇒ Integer

Get current cache size

Returns:

  • (Integer) —

    Number of entries in cache



232
233
234
235
236
 
# File 'lib/langfuse/prompt_cache.rb', line 232

def size
  @monitor.synchronize do
    @cache.size
  end
end

#stats ⇒ Hash

Returns Prompt cache statistics.

Returns:

  • (Hash) —

    Prompt cache statistics



203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/langfuse/prompt_cache.rb', line 203

def stats
  @monitor.synchronize do
    counts = count_entries_by_generation
    {
      backend: CacheBackend::MEMORY,
      enabled: true,
      current_generation_entries: counts.fetch(:current),
      orphaned_entries: counts.fetch(:orphaned),
      total_entries: @cache.size,
      global_generation: @global_generation,
      unsupported_counts: []
    }
  end
end

#storage_key(logical_key, name:) ⇒ String

Build a generated storage key for the current cache generation.

Parameters:

  • logical_key (String) —

    Stable logical cache identity

  • name (String) —

    Prompt name

Returns:

  • (String) —

    Generated storage key



191
192
193
194
195
196
197
198
199
200
 
# File 'lib/langfuse/prompt_cache.rb', line 191

def storage_key(logical_key, name:)
  @monitor.synchronize do
    self.class.storage_key(
      logical_key,
      name: name,
      global_generation: @global_generation,
      name_generation: @name_generations.fetch(name.to_s, 0)
    )
  end
end

#validate! ⇒ Boolean

Validate that the memory cache backend is usable.

rubocop:disable Naming/PredicateMethod

Returns:

  • (Boolean) 


251
252
253
 
# File 'lib/langfuse/prompt_cache.rb', line 251

def validate!
  true
end