Class: Kabosu::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/kabosu.rb

Overview

── Tokenizer: wrap output in MorphemeList ──

Instance Method Summary collapse

Instance Method Details

#_tokenize ⇒ Object 



173
 
# File 'lib/kabosu.rb', line 173

alias_method :_tokenize, :tokenize

#tokenize(text) ⇒ Object 



175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/kabosu.rb', line 175

def tokenize(text)
  unless text.is_a?(String)
    raise ArgumentError, "text must be a String"
  end

  batch = _tokenize(text)
  cost = batch.respond_to?(:internal_cost) ? batch.internal_cost : nil
  MorphemeList.new(batch, internal_cost: cost)
rescue RuntimeError => e
  raise TokenizationError.new(e.message), cause: e
end