Module: Engram::EmbeddingMetadata

Defined in:
lib/engram/embedding_metadata.rb

Overview

Helpers for storing and validating embedding provenance under Engram's reserved metadata namespace.

Constant Summary collapse

RESERVED_KEY = 
"_engram"
EMBEDDING_KEY = 
"embedding"

Class Method Summary collapse

Class Method Details

.attach(record, embedder: nil, embedding_metadata: nil) ⇒ Object 



37
38
39
40
41
42
 
# File 'lib/engram/embedding_metadata.rb', line 37

def attach(record, embedder: nil, embedding_metadata: nil)
  metadata = stringify_keys(embedding_metadata || for_embedder(embedder, embedding: record.embedding)) || {}
  return record if metadata.empty?

  record.with(metadata: merge(record.metadata, metadata))
end

.build(adapter:, dimensions:, model: nil, provider: nil) ⇒ Object 



14
15
16
17
18
19
20
21
22
23
 
# File 'lib/engram/embedding_metadata.rb', line 14

def build(adapter:, dimensions:, model: nil, provider: nil)
  data = {
    "adapter" => adapter.to_s,
    "dimensions" => dimensions
  }
  data["provider"] = provider.to_s if provider
  data["model"] = model.to_s if model
  data["fingerprint"] = fingerprint(data)
  data
end

.extract(metadata) ⇒ Object 



44
45
46
47
48
49
50
 
# File 'lib/engram/embedding_metadata.rb', line 44

def extract(metadata)
  metadata = stringify_keys(metadata || {})
  reserved = metadata[RESERVED_KEY]
  return {} unless reserved.is_a?(Hash)

  stringify_keys(reserved[EMBEDDING_KEY] || {})
end

.fingerprint(metadata) ⇒ Object 



114
115
116
117
 
# File 'lib/engram/embedding_metadata.rb', line 114

def fingerprint(metadata)
  relevant = stringify_keys(metadata).values_at("adapter", "provider", "model", "dimensions")
  Digest::SHA256.hexdigest(relevant.map { |value| value.nil? ? "" : value.to_s }.join("\0"))
end

.for_embedder(embedder, embedding: nil) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/engram/embedding_metadata.rb', line 25

def for_embedder(embedder, embedding: nil)
  return nil unless embedder.respond_to?(:embedding_metadata)

  metadata = stringify_keys(embedder.embedding_metadata || {})
  return nil if metadata.empty?

  dimensions = embedding.respond_to?(:length) ? embedding.length : metadata["dimensions"]
  metadata = metadata.merge("dimensions" => dimensions) if dimensions
  metadata["fingerprint"] = fingerprint(metadata)
  metadata
end

.merge(metadata, embedding_metadata) ⇒ Object 



52
53
54
55
56
 
# File 'lib/engram/embedding_metadata.rb', line 52

def merge(metadata, embedding_metadata)
  Engram::ReservedMetadata.attach(
    metadata, EMBEDDING_KEY, embedding_metadata || {}, namespace_description: "embedding metadata"
  )
end

.search(store, embedding:, embedding_metadata:, scope:, limit:, kinds: nil) ⇒ Object 



58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/engram/embedding_metadata.rb', line 58

def search(store, embedding:, embedding_metadata:, scope:, limit:, kinds: nil)
  store.search(
    embedding: embedding,
    embedding_metadata: embedding_metadata,
    scope: scope,
    limit: limit,
    kinds: kinds
  )
rescue ArgumentError => error
  raise unless unknown_embedding_metadata_keyword?(error)

  store.search(embedding: embedding, scope: scope, limit: limit, kinds: kinds)
end

.stringify_keys(value) ⇒ Object 



119
120
121
122
123
124
125
126
127
128
 
# File 'lib/engram/embedding_metadata.rb', line 119

def stringify_keys(value)
  case value
  when Hash
    value.each_with_object({}) do |(key, nested), out|
      out[key.to_s] = stringify_keys(nested)
    end
  else
    value
  end
end

.unknown_embedding_metadata_keyword?(error) ⇒ Boolean

Returns:

  • (Boolean) 


130
131
132
133
 
# File 'lib/engram/embedding_metadata.rb', line 130

def unknown_embedding_metadata_keyword?(error)
  error.message.include?("unknown keyword: :embedding_metadata") ||
    error.message.include?("unknown keyword: \"embedding_metadata\"")
end

.validate_query!(embedding, embedding_metadata) ⇒ Object

Raises:



72
73
74
75
76
77
78
79
80
81
 
# File 'lib/engram/embedding_metadata.rb', line 72

def validate_query!(embedding, embedding_metadata)
  metadata = stringify_keys(embedding_metadata || {})
  return if metadata.empty?

  expected = metadata["dimensions"]
  return unless expected && embedding.respond_to?(:length) && embedding.length != expected.to_i

  raise Engram::Error,
    "embedding dimension mismatch: query vector has #{embedding.length} dimensions, metadata declares #{expected}"
end

.validate_record!(record, query_embedding, query_metadata) ⇒ Object

Raises:



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/engram/embedding_metadata.rb', line 83

def validate_record!(record, query_embedding, query_metadata)
  if query_embedding.respond_to?(:length) && record.embedding.respond_to?(:length) &&
      query_embedding.length != record.embedding.length
    raise Engram::Error,
      "embedding dimension mismatch: query vector has #{query_embedding.length} dimensions, " \
      "record #{record.id.inspect} has #{record.embedding.length}"
  end

  stored = extract(record.metadata)
  return if stored.empty?

  stored_dimensions = stored["dimensions"]
  if stored_dimensions && record.embedding.respond_to?(:length) && record.embedding.length != stored_dimensions.to_i
    raise Engram::Error,
      "embedding metadata mismatch: record #{record.id.inspect} has #{record.embedding.length} dimensions, " \
      "metadata declares #{stored_dimensions}"
  end

  query_metadata = stringify_keys(query_metadata || {})
  return if query_metadata.empty?

  conflicting_key = %w[adapter provider model dimensions fingerprint].find do |key|
    stored.key?(key) && query_metadata.key?(key) && stored[key].to_s != query_metadata[key].to_s
  end
  return unless conflicting_key

  raise Engram::Error,
    "embedding metadata mismatch for record #{record.id.inspect}: #{conflicting_key} " \
    "#{stored[conflicting_key].inspect} does not match query #{query_metadata[conflicting_key].inspect}"
end