Class: Mxrb::Semantic::TfidfEmbedder

Inherits:
Object
  • Object
show all
Defined in:
lib/mxrb/semantic/tfidf_embedder.rb

Overview

Deterministic hashed term-frequency embeddings with unit normalization.

Constant Summary collapse

DIM = 
512

Instance Method Summary collapse

Instance Method Details

#backend ⇒ Object 



10
 
# File 'lib/mxrb/semantic/tfidf_embedder.rb', line 10

def backend = :tfidf

#dimension ⇒ Object 



9
 
# File 'lib/mxrb/semantic/tfidf_embedder.rb', line 9

def dimension = DIM

#embed(text) ⇒ Object 



12
13
14
15
16
17
18
19
20
21
 
# File 'lib/mxrb/semantic/tfidf_embedder.rb', line 12

def embed(text)
  tokens = tokenize(text)
  return Array.new(DIM, 0.0) if tokens.empty?

  freq = tokens.tally
  total = tokens.size.to_f
  vec = Array.new(DIM, 0.0)
  freq.each { |term, count| vec[bucket(term)] += count / total }
  normalize(vec)
end