Class: Kotoshu::Languages::Persian::Tokenizer

Inherits:
Kotoshu::Language::Tokenizer::Base show all
Defined in:
lib/kotoshu/languages/fa/language.rb

Constant Summary collapse

PERSIAN_SEPARATORS = 
/[\s"()\[\]{}<>,.;:!?\\\/|`~@#$%^&*+\-ยทุŒุ›ุŸ]/

Instance Method Summary collapse

Methods inherited from Kotoshu::Language::Tokenizer::Base

#skip_token?, #tokenize_with_positions, #word_boundary_regex, #word_char?

Instance Method Details

#tokenize(text) ⇒ Object 



55
56
57
58
59
60
61
 
# File 'lib/kotoshu/languages/fa/language.rb', line 55

def tokenize(text)
  return [] if text.nil? || text.strip.empty?

  text.split(PERSIAN_SEPARATORS)
    .map { |token| normalize(token) }
    .reject { |token| skip_token?(token) }
end