Class: Langfuse::PromptCache

Inherits:
Object
  • Object
show all
Includes:
StaleWhileRevalidate
Defined in:
lib/langfuse/prompt_cache.rb

Overview

Simple in-memory cache for prompt data with TTL

Thread-safe cache implementation for storing prompt responses with time-to-live expiration.

rubocop:disable Metrics/ClassLength

Examples:

cache = Langfuse::PromptCache.new(ttl: 60)
cache.set("greeting:1", prompt_data)
cache.get("greeting:1") # => prompt_data

Defined Under Namespace

Classes: CacheEntry

Constant Summary collapse

MAX_NAME_GENERATIONS =

Caps the per-name generation map. Without a cap, long-lived processes that invalidate across many distinct prompts grow it unboundedly; LRU eviction keeps the working set live and lets cold names go.

1024

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from StaleWhileRevalidate

#fetch_with_stale_while_revalidate, #initialize_swr, #refresh_async, #shutdown, #swr_enabled?, #write_with_stale_while_revalidate

Constructor Details

#initialize(ttl: 60, max_size: 1000, stale_ttl: 0, refresh_threads: 5, logger: default_logger) ⇒ PromptCache

Initialize a new cache

Parameters:

  • ttl (Integer) (defaults to: 60) —

    Time-to-live in seconds (default: 60)

  • max_size (Integer) (defaults to: 1000) —

    Maximum cache size (default: 1000)

  • stale_ttl (Integer) (defaults to: 0) —

    Stale TTL for SWR in seconds (default: 0, SWR disabled). Note: :indefinite is normalized to 1000 years by Config before being passed here.

  • refresh_threads (Integer) (defaults to: 5) —

    Number of background refresh threads (default: 5)

  • logger (Logger, nil) (defaults to: default_logger) —

    Logger instance for error reporting (default: nil, creates new logger)



84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/langfuse/prompt_cache.rb', line 84

def initialize(ttl: 60, max_size: 1000, stale_ttl: 0, refresh_threads: 5, logger: default_logger)
  @ttl = ttl
  @max_size = max_size
  @stale_ttl = stale_ttl
  @logger = logger
  @cache = {}
  @global_generation = 0
  @name_generations = {}
  @name_generation_counter = 0
  @monitor = Monitor.new
  @locks = {} # Track locks for in-memory locking
  initialize_swr(refresh_threads: refresh_threads) if swr_enabled?
  ForkSafety.register(self)
end

Instance Attribute Details

#logger ⇒ Logger (readonly)

Returns Logger instance for error reporting.

Returns:

  • (Logger) —

    Logger instance for error reporting



74
75
76
 
# File 'lib/langfuse/prompt_cache.rb', line 74

def logger
  @logger
end

#max_size ⇒ Integer (readonly)

Returns Maximum number of cache entries.

Returns:

  • (Integer) —

    Maximum number of cache entries



68
69
70
 
# File 'lib/langfuse/prompt_cache.rb', line 68

def max_size
  @max_size
end

#stale_ttl ⇒ Integer (readonly)

Returns Stale TTL for SWR in seconds.

Returns:

  • (Integer) —

    Stale TTL for SWR in seconds



71
72
73
 
# File 'lib/langfuse/prompt_cache.rb', line 71

def stale_ttl
  @stale_ttl
end

#ttl ⇒ Integer (readonly)

Returns Time-to-live in seconds.

Returns:

  • (Integer) —

    Time-to-live in seconds



65
66
67
 
# File 'lib/langfuse/prompt_cache.rb', line 65

def ttl
  @ttl
end

Class Method Details

.build_key(name, version: nil, label: nil) ⇒ String

Build a cache key from prompt name and options

Parameters:

  • name (String) —

    Prompt name

  • version (Integer, nil) (defaults to: nil) —

    Optional version

  • label (String, nil) (defaults to: nil) —

    Optional label

Returns:

  • (String) —

    Cache key



267
268
269
270
271
272
273
 
# File 'lib/langfuse/prompt_cache.rb', line 267

def self.build_key(name, version: nil, label: nil)
  key = name.to_s
  key += ":v#{version}" if version
  key += ":#{label}" if label
  key += ":production" unless version || label
  key
end

.storage_key(logical_key, name:, global_generation:, name_generation:) ⇒ String

Build a generated storage key from generation metadata.

Parameters:

  • logical_key (String) —

    Stable logical cache identity

  • name (String) —

    Prompt name

  • global_generation (Integer) —

    Global cache generation

  • name_generation (Integer) —

    Prompt-name cache generation

Returns:

  • (String) —

    Generated storage key



282
283
284
285
 
# File 'lib/langfuse/prompt_cache.rb', line 282

def self.storage_key(logical_key, name:, global_generation:, name_generation:)
  encoded_name = Base64.urlsafe_encode64(name.to_s, padding: false)
  "g#{global_generation}:n#{encoded_name}:#{name_generation}:#{logical_key}"
end

Instance Method Details

#cleanup_expired ⇒ Integer

Remove expired entries from cache

Returns:

  • (Integer) —

    Number of entries removed



226
227
228
229
230
231
232
 
# File 'lib/langfuse/prompt_cache.rb', line 226

def cleanup_expired
  @monitor.synchronize do
    initial_size = @cache.size
    @cache.delete_if { |_key, entry| entry.expired? }
    initial_size - @cache.size
  end
end

#clear ⇒ void

This method returns an undefined value.

Clear the entire cache



155
156
157
158
159
 
# File 'lib/langfuse/prompt_cache.rb', line 155

def clear
  @monitor.synchronize do
    @cache.clear
  end
end

#clear_logically ⇒ Integer

Logically invalidate every generated storage key.

Returns:

  • (Integer) —

    New global generation



164
165
166
167
168
 
# File 'lib/langfuse/prompt_cache.rb', line 164

def clear_logically
  @monitor.synchronize do
    @global_generation += 1
  end
end

#delete(key) ⇒ Boolean

Delete one generated storage key.

Parameters:

  • key (String) —

    Generated storage key

Returns:

  • (Boolean) —

    true if an entry was removed



146
147
148
149
150
 
# File 'lib/langfuse/prompt_cache.rb', line 146

def delete(key)
  @monitor.synchronize do
    !@cache.delete(key).nil?
  end
end

#empty? ⇒ Boolean

Check if cache is empty

Returns:

  • (Boolean) 


246
247
248
249
250
 
# File 'lib/langfuse/prompt_cache.rb', line 246

def empty?
  @monitor.synchronize do
    @cache.empty?
  end
end

#entry(key) ⇒ CacheEntry?

Read a raw cache entry, including stale entries.

Parameters:

  • key (String) —

    Cache key

Returns:



117
118
119
120
121
 
# File 'lib/langfuse/prompt_cache.rb', line 117

def entry(key)
  @monitor.synchronize do
    @cache[key]
  end
end

#get(key) ⇒ Object?

Get a value from the cache

Parameters:

  • key (String) —

    Cache key

Returns:

  • (Object, nil) —

    Cached value or nil if not found/expired



103
104
105
106
107
108
109
110
111
 
# File 'lib/langfuse/prompt_cache.rb', line 103

def get(key)
  @monitor.synchronize do
    entry = @cache[key]
    return nil unless entry
    return nil if entry.expired?

    entry.data
  end
end

#invalidate_name(name) ⇒ Integer

Logically invalidate every cache variant for one prompt name.

Generations come from a monotonic global counter, not a per-name counter, so an evicted name re-entering the map can't reuse a generation value that's still embedded in a stale @cache entry.

Parameters:

  • name (String) —

    Prompt name

Returns:

  • (Integer) —

    New name generation



178
179
180
181
182
183
184
185
186
 
# File 'lib/langfuse/prompt_cache.rb', line 178

def invalidate_name(name)
  @monitor.synchronize do
    name_str = name.to_s
    @name_generations.delete(name_str)
    @name_generations.shift if @name_generations.size >= MAX_NAME_GENERATIONS
    @name_generation_counter += 1
    @name_generations[name_str] = @name_generation_counter
  end
end

#set(key, value, ttl: nil, stale_ttl: nil) ⇒ Object

Set a value in the cache

Parameters:

  • key (String) —

    Cache key

  • value (Object) —

    Value to cache

Returns:

  • (Object) —

    The cached value



128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/langfuse/prompt_cache.rb', line 128

def set(key, value, ttl: nil, stale_ttl: nil)
  @monitor.synchronize do
    # Evict oldest entry if at max size
    evict_oldest if @cache.size >= max_size
    # TTL math is inlined (not extracted to a helper) to keep this hot path
    # allocation-free apart from the CacheEntry below.
    effective_ttl = ttl.nil? ? self.ttl : ttl
    effective_stale_ttl = stale_ttl.nil? ? self.stale_ttl : stale_ttl
    fresh_until = Time.now + effective_ttl
    @cache[key] = CacheEntry.new(value, fresh_until, fresh_until + effective_stale_ttl)
    value
  end
end

#size ⇒ Integer

Get current cache size

Returns:

  • (Integer) —

    Number of entries in cache



237
238
239
240
241
 
# File 'lib/langfuse/prompt_cache.rb', line 237

def size
  @monitor.synchronize do
    @cache.size
  end
end

#stats ⇒ Hash

Returns Prompt cache statistics.

Returns:

  • (Hash) —

    Prompt cache statistics



205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
 
# File 'lib/langfuse/prompt_cache.rb', line 205

def stats
  @monitor.synchronize do
    counts = count_entries_by_generation
    {
      backend: CacheBackend::MEMORY,
      enabled: true,
      current_generation_entries: counts.fetch(:current),
      orphaned_entries: counts.fetch(:orphaned),
      total_entries: @cache.size,
      ttl: ttl,
      size: @cache.size,
      max_size: max_size,
      global_generation: @global_generation,
      unsupported_counts: []
    }
  end
end

#storage_key(logical_key, name:) ⇒ String

Build a generated storage key for the current cache generation.

Parameters:

  • logical_key (String) —

    Stable logical cache identity

  • name (String) —

    Prompt name

Returns:

  • (String) —

    Generated storage key



193
194
195
196
197
198
199
200
201
202
 
# File 'lib/langfuse/prompt_cache.rb', line 193

def storage_key(logical_key, name:)
  @monitor.synchronize do
    self.class.storage_key(
      logical_key,
      name: name,
      global_generation: @global_generation,
      name_generation: @name_generations.fetch(name.to_s, 0)
    )
  end
end

#validate! ⇒ Boolean

Validate that the memory cache backend is usable.

rubocop:disable Naming/PredicateMethod

Returns:

  • (Boolean) 


256
257
258
 
# File 'lib/langfuse/prompt_cache.rb', line 256

def validate!
  true
end