Class: Kotoshu::Embeddings::Vocabulary

Inherits:
Object
  • Object
show all
Includes:
VocabularyProtocol
Defined in:
lib/kotoshu/embeddings/vocabulary.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Protocol

#assert_implemented_by!, #compliance_errors, #optional, #optional_methods, #required, #required_methods

Constructor Details

#initialize(language_code:, word_to_index:) ⇒ Vocabulary

Create a new vocabulary

Parameters:

  • language_code (String) —

    ISO 639-1 language code

  • word_to_index (Hash{String => Integer}) —

    Word to index mapping

Raises:

  • (ArgumentError) —

    If word_to_index is empty



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 40

def initialize(language_code:, word_to_index:)
  if word_to_index.nil? || word_to_index.empty?
    raise ArgumentError,
          'word_to_index cannot be empty'
  end

  @language_code = language_code
  @word_to_index = word_to_index.dup.freeze

  # Build reverse index (index -> word)
  @index_to_word = Array.new(@word_to_index.size)
  @word_to_index.each do |word, index|
    @index_to_word[index] = word if index < @index_to_word.size
  end
  @index_to_word.freeze
end

Instance Attribute Details

#index_to_word ⇒ Array<String> (readonly)

Returns Index to word mapping (sparse array).

Returns:

  • (Array<String>) —

    Index to word mapping (sparse array)



31
32
33
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 31

def index_to_word
  @index_to_word
end

#language_code ⇒ String (readonly)

Returns ISO 639-1 language code.

Returns:

  • (String) —

    ISO 639-1 language code



25
26
27
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 25

def language_code
  @language_code
end

#word_to_index ⇒ Hash{String => Integer} (readonly)

Returns Word to index mapping.

Returns:

  • (Hash{String => Integer}) —

    Word to index mapping



28
29
30
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 28

def word_to_index
  @word_to_index
end

Class Method Details

.detect_language_from_path(path) ⇒ String

Detect language code from file path

Parameters:

  • path (String) —

    File path

Returns:

  • (String) —

    Detected language code



246
247
248
249
250
251
252
253
254
255
256
257
258
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 246

def self.detect_language_from_path(path)
  basename = File.basename(path)

  if basename =~ /(\w+)\.vocab\.json\z/
    return $1
  end

  if basename =~ /\.(\w+)\.vocab\.json\z/
    return $1
  end

  'unknown'
end

.from_cache(language_code, cache: nil) ⇒ Vocabulary?

Load vocabulary for a language from the model cache.

Resolves the vocab.json sibling of the cached ONNX model. Returns nil when either the cache or the vocab file is unavailable so callers can degrade gracefully.

Parameters:

  • language_code (String) —

    ISO 639-1 language code

  • cache (Kotoshu::Cache::ModelCache, nil) (defaults to: nil) —

    Optional cache instance

Returns:



269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 269

def self.from_cache(language_code, cache: nil)
  cache ||= Kotoshu::Cache::ModelCache.new
  onnx_path = cache.get_onnx_model(language_code)
  return nil unless onnx_path

  vocab_path = File.join(File.dirname(onnx_path),
                         "fasttext.#{language_code}.vocab.json")
  return nil unless File.file?(vocab_path)

  from_file(vocab_path, language_code: language_code)
end

.from_file(path, language_code: nil) ⇒ Vocabulary

Load vocabulary from JSON file

Parameters:

  • path (String) —

    Path to JSON file

  • language_code (String) (defaults to: nil) —

    Language code (auto-detected from filename if nil)

Returns:

Raises:

  • (ArgumentError) —

    If file doesn't exist

  • (Json::ParserError) —

    If file is not valid JSON



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 137

def self.from_file(path, language_code: nil)
  raise ArgumentError, "File not found: #{path}" unless File.exist?(path)

  language_code ||= detect_language_from_path(path)

  data = JSON.parse(File.read(path))

  case data
  when Hash
    word_to_index = data.transform_keys(&:freeze).freeze
  when Array
    word_to_index = {}
    data.each_with_index do |word, index|
      word_to_index[word.freeze] = index
    end
    word_to_index.freeze
  else
    raise ArgumentError, 'Invalid vocabulary format: expected Hash or Array'
  end

  new(language_code: language_code, word_to_index: word_to_index)
end

.from_words(words, language_code: 'en') ⇒ Vocabulary

Create vocabulary from Array of words

Parameters:

  • words (Array<String>) —

    Array of words

  • language_code (String) (defaults to: 'en') —

    Language code

Returns:



166
167
168
169
170
171
172
173
174
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 166

def self.from_words(words, language_code: 'en')
  word_to_index = {}
  words.each_with_index do |word, index|
    word_to_index[word.freeze] = index
  end
  word_to_index.freeze

  new(language_code: language_code, word_to_index: word_to_index)
end

Instance Method Details

#common_words(n: 10) ⇒ Array<String>

Get common/most frequent words

Parameters:

  • n (Integer) (defaults to: 10) —

    Number of words to return

Returns:

  • (Array<String>) —

    Array of common words



106
107
108
109
110
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 106

def common_words(n: 10)
  return [] if @word_to_index.empty?

  @word_to_index.keys.first(n)
end

#empty? ⇒ Boolean

Check if vocabulary is empty

Returns:

  • (Boolean) —

    True if empty



199
200
201
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 199

def empty?
  @word_to_index.empty?
end

#get_word(index) ⇒ String?

Get word by index

Parameters:

  • index (Integer) —

    The index to look up

Returns:

  • (String, nil) —

    Word at the index, or nil if not found



71
72
73
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 71

def get_word(index)
  @index_to_word[index]
end

#include?(word) ⇒ Boolean

Check if word exists in vocabulary

Parameters:

  • word (String) —

    Word to check

Returns:

  • (Boolean) —

    True if word exists



80
81
82
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 80

def include?(word)
  @word_to_index.key?(word)
end

#lookup(word) ⇒ Integer?

Look up word index

Parameters:

  • word (String) —

    The word to look up

Returns:

  • (Integer, nil) —

    Index of the word, or nil if not found



62
63
64
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 62

def lookup(word)
  @word_to_index[word]
end

#sample(n: 10) ⇒ Array<String>

Get a sample of words

Parameters:

  • n (Integer) (defaults to: 10) —

    Number of words to sample

Returns:

  • (Array<String>) —

    Sample of words



208
209
210
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 208

def sample(n: 10)
  @word_to_index.keys.sample(n)
end

#save_to_file(path, format: :hash) ⇒ Object

Save vocabulary to JSON file

Parameters:

  • path (String) —

    Path to save file

  • format (Symbol) (defaults to: :hash) —

    Format: :hash or :array



181
182
183
184
185
186
187
188
189
190
191
192
193
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 181

def save_to_file(path, format: :hash)
  case format
  when :hash
    data = @word_to_index.dup
  when :array
    max_index = @index_to_word.compact.length
    data = @index_to_word.compact.first(max_index)
  else
    raise ArgumentError, "Unknown format: #{format}"
  end

  File.write(path, JSON.pretty_generate(data))
end

#size ⇒ Integer

Get vocabulary size

Returns:

  • (Integer) —

    Number of words in vocabulary



88
89
90
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 88

def size
  @word_to_index.size
end

#sub_vocabulary(words) ⇒ Vocabulary

Create a sub-vocabulary containing only specified words

Parameters:

  • words (Array<String>) —

    Words to include

Returns:

  • (Vocabulary) —

    New vocabulary with subset of words



217
218
219
220
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 217

def sub_vocabulary(words)
  filtered = @word_to_index.slice(*words)
  self.class.new(language_code: @language_code, word_to_index: filtered)
end

#to_h ⇒ Hash{String => Integer}

Convert to Hash

Returns:

  • (Hash{String => Integer}) —

    Copy of word_to_index mapping



116
117
118
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 116

def to_h
  @word_to_index.dup
end

#to_s ⇒ String Also known as: inspect

String representation

Returns:

  • (String) 


236
237
238
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 236

def to_s
  "Vocabulary(language: #{@language_code}, size: #{@word_to_index.size})"
end

#valid_index?(index) ⇒ Boolean

Check if index is valid

Parameters:

  • index (Integer) —

    Index to check

Returns:

  • (Boolean) —

    True if index is valid



97
98
99
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 97

def valid_index?(index)
  index.is_a?(Integer) && index >= 0 && index < @word_to_index.size
end

#words ⇒ Enumerator<String>

Get all words as enumerator

Returns:

  • (Enumerator<String>) —

    Enumerator of all words



124
125
126
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 124

def words
  @word_to_index.each_key
end

#words_starting_with(prefix) ⇒ Array<String>

Find words starting with a prefix

Parameters:

  • prefix (String) —

    Prefix to match

Returns:

  • (Array<String>) —

    Matching words



227
228
229
230
 
# File 'lib/kotoshu/embeddings/vocabulary.rb', line 227

def words_starting_with(prefix)
  pattern = /^#{Regexp.escape(prefix)}/
  @word_to_index.keys.grep(pattern)
end