Class: Kabosu::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/kabosu.rb

Overview

── Tokenizer: wrap output in MorphemeList ──

Instance Method Summary collapse

Instance Method Details

#_tokenize ⇒ Object 



152
 
# File 'lib/kabosu.rb', line 152

alias_method :_tokenize, :tokenize

#tokenize(text) ⇒ Object 



154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/kabosu.rb', line 154

def tokenize(text)
  unless text.is_a?(String)
    raise ArgumentError, "text must be a String"
  end

  batch = _tokenize(text)
  cost = batch.respond_to?(:internal_cost) ? batch.internal_cost : nil
  MorphemeList.new(batch, internal_cost: cost)
rescue RuntimeError => e
  raise TokenizationError.new(e.message), cause: e
end