Class: Jekyll::FastReader::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/jekyll/fast_reader/tokenizer.rb

Constant Summary collapse

PATTERN = 
/(#{Characters::WORD})|(\s+)|([^\sa-zA-Z]+)/

Class Method Summary collapse

Class Method Details

.tokenize(text) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
 
# File 'lib/jekyll/fast_reader/tokenizer.rb', line 6

def self.tokenize(text)
  text.scan(PATTERN).map do |word, whitespace, punctuation|
    if word
      [:word, word]
    elsif whitespace
      [:whitespace, whitespace]
    else
      [:punctuation, punctuation]
    end
  end
end