Class: Kotoshu::Embeddings::SimilarityEngine

Inherits:
Object
  • Object
show all
Includes:
SimilarityEngineProtocol
Defined in:
lib/kotoshu/embeddings/similarity_engine.rb

Constant Summary collapse

DEFAULT_CACHE_SIZE =

Default embedding dimension for norm cache initialization

10_000

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Protocol

#assert_implemented_by!, #compliance_errors, #optional, #optional_methods, #required, #required_methods

Constructor Details

#initialize(pre_normalize: false, cache_norms: true) ⇒ SimilarityEngine

Create a new similarity engine

Parameters:

  • pre_normalize (Boolean) (defaults to: false) —

    Whether to pre-normalize vectors

  • cache_norms (Boolean) (defaults to: true) —

    Whether to cache vector norms



40
41
42
43
44
45
46
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 40

def initialize(pre_normalize: false, cache_norms: true)
  @pre_normalize = pre_normalize
  @cache_norms = cache_norms
  @norm_cache = cache_norms ? {} : nil
  @cache_hits = 0
  @cache_misses = 0
end

Instance Attribute Details

#cache_hits ⇒ Integer (readonly)

Returns Number of cache hits.

Returns:

  • (Integer) —

    Number of cache hits



30
31
32
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 30

def cache_hits
  @cache_hits
end

#cache_misses ⇒ Integer (readonly)

Returns Number of cache misses.

Returns:

  • (Integer) —

    Number of cache misses



33
34
35
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 33

def cache_misses
  @cache_misses
end

Instance Method Details

#cache_stats ⇒ Hash

Get cache statistics

Returns:

  • (Hash) —

    Cache statistics



187
188
189
190
191
192
193
194
195
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 187

def cache_stats
  total = @cache_hits + @cache_misses
  {
    hits: @cache_hits,
    misses: @cache_misses,
    hit_rate: total.zero? ? 0.0 : @cache_hits.to_f / total,
    cache_size: @norm_cache&.size || 0
  }
end

#clear_cache ⇒ self

Clear the norm cache

Returns:

  • (self) 


176
177
178
179
180
181
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 176

def clear_cache
  @norm_cache&.clear
  @cache_hits = 0
  @cache_misses = 0
  self
end

#compute_all_pairs(vectors) ⇒ Array<Array<Float>>

Compute all pairwise similarities for a set of vectors

Parameters:

  • vectors (Array<Array<Float>>) —

    ] Array of vectors

Returns:

  • (Array<Array<Float>>) —

    Similarity matrix



213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 213

def compute_all_pairs(vectors)
  n = vectors.length
  matrix = Array.new(n) { Array.new(n, 0.0) }

  (0...n).each do |i|
    matrix[i][i] = 1.0
    ((i + 1)...n).each do |j|
      sim = cosine(vectors[i], vectors[j])
      matrix[i][j] = sim
      matrix[j][i] = sim
    end
  end

  matrix
end

#cosine(vec1, vec2) ⇒ Float

Compute cosine similarity between two vectors

Cosine similarity = dot(v1, v2) / (||v1|| * ||v2||) Range: -1.0 (opposite) to 1.0 (identical)

Parameters:

  • vec1 (Array<Float>) —

    First vector

  • vec2 (Array<Float>) —

    Second vector

Returns:

  • (Float) —

    Cosine similarity, or 0.0 if either vector is nil/empty



57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 57

def cosine(vec1, vec2)
  return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?

  validate_dimensions!(vec1, vec2)

  norm1 = get_norm(vec1)
  norm2 = get_norm(vec2)

  return 0.0 if norm1.zero? || norm2.zero?

  dot = dot_product(vec1, vec2)
  dot / (norm1 * norm2)
end

#cosine_batch(pairs) ⇒ Array<Float>

Compute similarity for a batch of vector pairs

More efficient than calling cosine() repeatedly.

Parameters:

  • pairs (Array<Array<Array<Float>>>) —

    Array of [vec1, vec2] pairs

Returns:

  • (Array<Float>) —

    Array of similarities



204
205
206
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 204

def cosine_batch(pairs)
  pairs.map { |v1, v2| cosine(v1, v2) }
end

#dot_product(vec1, vec2) ⇒ Float

Compute dot product between two vectors

Parameters:

  • vec1 (Array<Float>) —

    First vector

  • vec2 (Array<Float>) —

    Second vector

Returns:

  • (Float) —

    Dot product



77
78
79
80
81
82
83
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 77

def dot_product(vec1, vec2)
  return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?

  validate_dimensions!(vec1, vec2)

  vec1.zip(vec2).sum { |a, b| a * b }
end

#euclidean(vec1, vec2) ⇒ Float

Compute Euclidean distance between two vectors

Parameters:

  • vec1 (Array<Float>) —

    First vector

  • vec2 (Array<Float>) —

    Second vector

Returns:

  • (Float) —

    Euclidean distance



91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 91

def euclidean(vec1, vec2)
  return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?
  return 0.0 if vec1.equal?(vec2)

  validate_dimensions!(vec1, vec2)

  sum = 0.0
  vec1.zip(vec2) do |a, b|
    diff = a - b
    sum += diff * diff
  end
  Math.sqrt(sum)
end

#is_normalized?(vec) ⇒ Boolean

Check if vectors are normalized (unit length)

Parameters:

  • vec (Array<Float>) —

    Vector to check

Returns:

  • (Boolean) —

    True if vector is normalized



157
158
159
160
161
162
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 157

def is_normalized?(vec)
  return true if vec.nil? || vec.empty?

  norm = get_norm(vec)
  (norm - 1.0).abs < Float::EPSILON * 10
end

#manhattan(vec1, vec2) ⇒ Float

Compute Manhattan (L1) distance between two vectors

Parameters:

  • vec1 (Array<Float>) —

    First vector

  • vec2 (Array<Float>) —

    Second vector

Returns:

  • (Float) —

    Manhattan distance



111
112
113
114
115
116
117
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 111

def manhattan(vec1, vec2)
  return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?

  validate_dimensions!(vec1, vec2)

  vec1.zip(vec2).sum { |a, b| (a - b).abs }
end

#normalization_required? ⇒ Boolean

Check if normalization is required for accurate similarity

Returns:

  • (Boolean) —

    True if normalization should be applied



168
169
170
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 168

def normalization_required?
  !@pre_normalize
end

#normalize_and_compute(vec1, vec2) ⇒ Float

Normalize and compute similarity in one pass

For pre-normalized vectors, this is just dot product (much faster).

Parameters:

  • vec1 (Array<Float>) —

    First vector

  • vec2 (Array<Float>) —

    Second vector

Returns:

  • (Float) —

    Cosine similarity



141
142
143
144
145
146
147
148
149
150
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 141

def normalize_and_compute(vec1, vec2)
  return 0.0 if vec1.nil? || vec2.nil? || vec1.empty? || vec2.empty?

  if @pre_normalize
    # For normalized vectors, cosine similarity = dot product
    dot_product(vec1, vec2)
  else
    cosine(vec1, vec2)
  end
end

#pre_normalize(vec) ⇒ Array<Float>

Pre-normalize a vector to unit length

Parameters:

  • vec (Array<Float>) —

    Vector to normalize

Returns:

  • (Array<Float>) —

    Normalized vector



124
125
126
127
128
129
130
131
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 124

def pre_normalize(vec)
  return vec.dup if vec.nil? || vec.empty?

  norm = get_norm(vec)
  return vec.dup if norm.zero?

  vec.map { |x| x / norm }
end

#pre_normalize? ⇒ Boolean

Returns Whether vectors are pre-normalized.

Returns:

  • (Boolean) —

    Whether vectors are pre-normalized



25
26
27
 
# File 'lib/kotoshu/embeddings/similarity_engine.rb', line 25

def pre_normalize?
  @pre_normalize
end