Class: Rjq::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/rjq/lexer.rb

Constant Summary collapse

KEYWORDS = 
%w[
  if then elif else end as def reduce foreach try catch label import include module
  null true false and or not break
].freeze
TWO_CHAR_OPERATORS = 
['//', '==', '!=', '<=', '>=', '|=', '+=', '-=', '*=', '/=', '%=', '..'].freeze
THREE_CHAR_OPERATORS = 
['//='].freeze
SINGLE_CHAR_TOKENS = 
{
  '.' => :dot,
  '|' => :pipe,
  ',' => :comma,
  '(' => :lparen,
  ')' => :rparen,
  '[' => :lbracket,
  ']' => :rbracket,
  '{' => :lbrace,
  '}' => :rbrace,
  ':' => :colon,
  ';' => :semicolon,
  '?' => :question,
  '+' => :operator,
  '-' => :operator,
  '*' => :operator,
  '/' => :operator,
  '%' => :operator,
  '=' => :operator,
  '<' => :operator,
  '>' => :operator
}.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, allow_comments: true, source_name: '<top-level>', initial_line: 1, initial_column: 1, start_offset: 0) ⇒ Lexer

Returns a new instance of Lexer.



40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/rjq/lexer.rb', line 40

def initialize(source, allow_comments: true, source_name: '<top-level>', initial_line: 1, initial_column: 1,
               start_offset: 0)
  @source = source.to_s
  @allow_comments = allow_comments
  @source_name = source_name
  @offset_base = start_offset
  @index = 0
  @line = initial_line
  @column = initial_column
  @tokens = []
end

Instance Attribute Details

#tokens ⇒ Object (readonly)

Returns the value of attribute tokens.



38
39
40
 
# File 'lib/rjq/lexer.rb', line 38

def tokens
  @tokens
end

Instance Method Details

#tokenize ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/rjq/lexer.rb', line 52

def tokenize
  until eof?
    skip_space
    break if eof?

    start_line = @line
    start_column = @column
    start_offset = @index
    char = current
    token =
      if char == '#'
        unless @allow_comments
          raise ParseError,
                "comments are disabled at line #{start_line}, column #{start_column}"
        end

        skip_comment
        next
      elsif char == '"'
        Token.new(type: :string, value: read_string, line: start_line, column: start_column)
      elsif char == '$'
        advance
        Token.new(type: :variable, value: read_identifier, line: start_line, column: start_column)
      elsif char == '@'
        advance
        Token.new(type: :format, value: read_identifier, line: start_line, column: start_column)
      elsif number_start?(char) || (char == '.' && digit?(@source[@index + 1]))
        Token.new(type: :number, value: read_number, line: start_line, column: start_column)
      elsif identifier_start?(char)
        identifier = read_identifier
        type = KEYWORDS.include?(identifier) ? :keyword : :identifier
        Token.new(type: type, value: identifier, line: start_line, column: start_column)
      else
        read_operator_or_punctuation(start_line, start_column)
      end
    if token
      token.start_offset = @offset_base + start_offset
      token.end_offset = @offset_base + @index
      token.filename = @source_name
      @tokens << token
    end
  end
  offset = @offset_base + @index
  @tokens << Token.new(type: :eof, value: nil, line: @line, column: @column, start_offset: offset,
                       end_offset: offset, filename: @source_name)
  @tokens
end