Module: MilkTea::Lexer::Symbols

Included in:
MilkTea::Lexer
Defined in:
lib/milk_tea/core/lexer/symbols.rb

Overview

Operator / punctuation lexing and grouping-delimiter depth tracking.

Instance Method Summary collapse

Instance Method Details

#adjust_grouping_depth(type, line_number, column) ⇒ Object 



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/milk_tea/core/lexer/symbols.rb', line 45

def adjust_grouping_depth(type, line_number, column)
  case type
  when :lparen, :lbracket
    if @grouping_depth.zero?
      @grouping_start_line = line_number
      @grouping_start_column = column
    end
    @grouping_depth += 1
  when :rparen, :rbracket
    @grouping_depth -= 1
    if @grouping_depth.negative?
      error = LexError.new("unexpected closing delimiter", line: line_number, column: column, path: @path)
      if @recovery_errors
        @recovery_errors << error
        @grouping_depth = 0
        return
      end

      raise error
    end
  end
end

#lex_symbol(line, index, line_number, line_offset:) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/milk_tea/core/lexer/symbols.rb', line 7

def lex_symbol(line, index, line_number, line_offset:)
  start = index
  lexeme = line[index, 3]
  if lexeme && THREE_CHAR_TOKENS.key?(lexeme)
    type = THREE_CHAR_TOKENS.fetch(lexeme)
    @tokens << token(type, lexeme, nil, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 3)
    return index + 3
  end

  lexeme = line[index, 2]
  if lexeme && TWO_CHAR_TOKENS.key?(lexeme)
    type = TWO_CHAR_TOKENS.fetch(lexeme)
    @tokens << token(type, lexeme, nil, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 2)
    adjust_grouping_depth(type, line_number, start + 1)
    return index + 2
  end

  lexeme = line[index]
  type = ONE_CHAR_TOKENS[lexeme]
  unless type
    if @recovery_errors && lexeme == "}"
      @recovery_errors << LexError.new("unexpected closing delimiter", line: line_number, column: start + 1, path: @path)
      return index + 1
    end

    if @recovery_errors
      @recovery_errors << LexError.new("unexpected character #{lexeme.inspect}", line: line_number, column: start + 1, path: @path)
      return index + 1
    end

    raise LexError.new("unexpected character #{lexeme.inspect}", line: line_number, column: start + 1, path: @path)
  end

  @tokens << token(type, lexeme, nil, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index + 1)
  adjust_grouping_depth(type, line_number, start + 1)
  index + 1
end