Class: RubyLlmMesh::Cache::SemanticCache

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm_mesh/cache/semantic_cache.rb

Overview

Distributed (or process-local) semantic response cache. Embeds prompts via Rag::Embeddings and matches with cosine similarity.

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config: RubyLlmMesh.configuration, store: nil, embedder: nil) ⇒ SemanticCache

Returns a new instance of SemanticCache.



26
27
28
29
30
 
# File 'lib/ruby_llm_mesh/cache/semantic_cache.rb', line 26

def initialize(config: RubyLlmMesh.configuration, store: nil, embedder: nil)
  @config = config
  @embedder = embedder || Rag::Embeddings.new(dimensions: config.semantic_cache_dimensions)
  @store = store || build_store
end

Instance Attribute Details

#embedder ⇒ Object (readonly)

Returns the value of attribute embedder.



24
25
26
 
# File 'lib/ruby_llm_mesh/cache/semantic_cache.rb', line 24

def embedder
  @embedder
end

#store ⇒ Object (readonly)

Returns the value of attribute store.



24
25
26
 
# File 'lib/ruby_llm_mesh/cache/semantic_cache.rb', line 24

def store
  @store
end

Class Method Details

.instance(config: RubyLlmMesh.configuration) ⇒ Object 



14
15
16
 
# File 'lib/ruby_llm_mesh/cache/semantic_cache.rb', line 14

def instance(config: RubyLlmMesh.configuration)
  @instance ||= new(config: config)
end

.reset! ⇒ Object 



18
19
20
21
 
# File 'lib/ruby_llm_mesh/cache/semantic_cache.rb', line 18

def reset!
  @instance&.clear!
  @instance = nil
end

Instance Method Details

#clear! ⇒ Object 



93
94
95
 
# File 'lib/ruby_llm_mesh/cache/semantic_cache.rb', line 93

def clear!
  @store.clear!
end

#enabled? ⇒ Boolean

Returns:

  • (Boolean) 


32
33
34
 
# File 'lib/ruby_llm_mesh/cache/semantic_cache.rb', line 32

def enabled?
  !!@config.semantic_cache_enabled
end

#lookup(prompt, system: nil) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/ruby_llm_mesh/cache/semantic_cache.rb', line 36

def lookup(prompt, system: nil)
  return nil unless enabled?

  query = cache_key_text(prompt, system)
  query_vec = @embedder.embed(query)
  threshold = @config.semantic_cache_threshold
  best = nil
  best_score = -1.0

  @store.all.each do |entry|
    next if entry.expires_at && entry.expires_at <= Time.now
    next unless entry.embedding.is_a?(Array) && entry.embedding.length == query_vec.length

    score = @embedder.cosine_similarity(query_vec, entry.embedding)
    if score >= threshold && score > best_score
      best = entry
      best_score = score
    end
  end

  return nil unless best

  payload = best.payload || {}
  Response.new(
    content: payload["content"] || payload[:content],
    provider: :semantic_cache,
    model: payload["model"] || payload[:model],
    usage: payload["usage"] || payload[:usage] || {},
    raw: { cache_id: best.id, similarity: best_score, original_provider: payload["provider"] },
    latency_ms: 0,
    fallback_used: false,
    cache_hit: true
  )
end

#store_response(prompt, response, system: nil) ⇒ Object 



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/ruby_llm_mesh/cache/semantic_cache.rb', line 71

def store_response(prompt, response, system: nil)
  return false unless enabled?
  return false if response.nil? || response.cache_hit

  query = cache_key_text(prompt, system)
  embedding = @embedder.embed(query)
  id = Digest::SHA256.hexdigest(query)[0, 32]
  payload = {
    "content" => response.content,
    "provider" => response.provider.to_s,
    "model" => response.model,
    "usage" => response.usage
  }
  @store.write(
    id: id,
    prompt: query,
    embedding: embedding,
    payload: payload,
    ttl: @config.semantic_cache_ttl
  )
end