Class: Tokenzr::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/tokenzr.rb

Instance Method Summary collapse

Instance Method Details

#digit_chars ⇒ Object 



27
28
29
 
# File 'lib/tokenzr.rb', line 27

def digit_chars
  @digit_chars ||= Set.new('0123456789'.chars)
end

#lone_chars ⇒ Object 



31
32
33
 
# File 'lib/tokenzr.rb', line 31

def lone_chars
  @lone_chars ||= Set.new('()[]<>{}!#$%&*+,-./:;=?@\\^`|~'.chars)
end

#parse(content) ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/tokenzr.rb', line 43

def parse(content)
  results = []
  current_token = nil
  enum = content.each_char
  @line = 1
  @column = 1
  @cur_line = 1
  @cur_col = 1

  while (chr = next_char(enum))
    start_line = @cur_line
    start_col = @cur_col

    if string_quotes.include?(chr)
      results << current_token unless current_token.nil?
      current_token = nil
      results << read_string(enum, chr, start_line, start_col)
      next
    end

    if space_chars.include?(chr)
      results << current_token unless current_token.nil?
      current_token = nil
      next
    end

    if digit_chars.include?(chr)
      if !current_token.nil? && current_token.type == :text
        # digits continue an identifier started by text/underscore
        current_token = Token.new(current_token.content + chr, :text, current_token.line, current_token.column)
        next
      end

      results << current_token unless current_token.nil?
      current_token = nil
      result = read_number(enum, chr, start_line, start_col)
      if result.is_a?(Array)
        results.concat(result)
      else
        results << result
      end
      next
    end

    if text_chars.include?(chr)
      if !current_token.nil? && current_token.type == :text
        current_token = Token.new(current_token.content + chr, :text, current_token.line, current_token.column)
      else
        results << current_token unless current_token.nil?
        current_token = Token.new(chr, :text, start_line, start_col)
      end
      next
    end

    if lone_chars.include?(chr)
      results << current_token unless current_token.nil?
      current_token = nil
      results << Token.new(chr, :lone, start_line, start_col)
      next
    end

    raise UnknownCharError, "Unknown character: #{chr.inspect}"
  end

  results << current_token unless current_token.nil?
  results
end

#space_chars ⇒ Object 



35
36
37
 
# File 'lib/tokenzr.rb', line 35

def space_chars
  @space_chars ||= Set.new(" \t\n\r\v\f".chars)
end

#string_quotes ⇒ Object 



39
40
41
 
# File 'lib/tokenzr.rb', line 39

def string_quotes
  @string_quotes ||= Set.new(%q{"'}.chars)
end

#text_chars ⇒ Object 



23
24
25
 
# File 'lib/tokenzr.rb', line 23

def text_chars
  @text_chars ||= Set.new('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_'.chars)
end