Class: MilkTea::Lexer

Inherits:
Object
  • Object
show all
Includes:
CharacterClasses, FormatStrings, Heredocs, Indentation, Numbers, Recovery, Strings, Symbols, Trivia
Defined in:
lib/milk_tea/core/lexer.rb,
lib/milk_tea/core/lexer/trivia.rb,
lib/milk_tea/core/lexer/numbers.rb,
lib/milk_tea/core/lexer/strings.rb,
lib/milk_tea/core/lexer/symbols.rb,
lib/milk_tea/core/lexer/heredocs.rb,
lib/milk_tea/core/lexer/recovery.rb,
lib/milk_tea/core/lexer/indentation.rb,
lib/milk_tea/core/lexer/format_strings.rb,
lib/milk_tea/core/lexer/character_classes.rb

Defined Under Namespace

Modules: CharacterClasses, FormatStrings, Heredocs, Indentation, Numbers, Recovery, Strings, Symbols, Trivia Classes: LexResult, StringLexResult, StringSegment

Constant Summary collapse

TOP_LEVEL_RESYNC_PREFIXES = 
%w[
  attribute const enum external flags foreign function include import interface
  link opaque public static_assert struct type union var variant extending event
].freeze
LINE_CONTINUATION_OPERATORS = 
%i[
  dot_dot
  plus minus star slash percent
  pipe amp caret
  or and
  equal_equal bang_equal
  less less_equal greater greater_equal
  shift_left shift_right
].freeze
THREE_CHAR_TOKENS = 
{
  "..." => :ellipsis,
  "<<=" => :shift_left_equal,
  ">>=" => :shift_right_equal,
}.freeze
TWO_CHAR_TOKENS = 
{
  "->" => :arrow,
  ".." => :dot_dot,
  "<<" => :shift_left,
  ">>" => :shift_right,
  "+=" => :plus_equal,
  "-=" => :minus_equal,
  "*=" => :star_equal,
  "/=" => :slash_equal,
  "%=" => :percent_equal,
  "&=" => :amp_equal,
  "|=" => :pipe_equal,
  "^=" => :caret_equal,
  "==" => :equal_equal,
  "!=" => :bang_equal,
  "<=" => :less_equal,
  ">=" => :greater_equal,
}.freeze
ONE_CHAR_TOKENS = 
{
  "&" => :amp,
  "@" => :at,
  ":" => :colon,
  "," => :comma,
  "^" => :caret,
  "." => :dot,
  "(" => :lparen,
  ")" => :rparen,
  "|" => :pipe,
  "[" => :lbracket,
  "]" => :rbracket,
  "?" => :question,
  "=" => :equal,
  "+" => :plus,
  "-" => :minus,
  "*" => :star,
  "/" => :slash,
  "%" => :percent,
  "<" => :less,
  ">" => :greater,
  "~" => :tilde,
}.freeze
INTEGER_SUFFIX_STRINGS = 
%w[ub us ul iz b s i u l z].sort_by { |s| -s.length }.freeze
IDENT_START_BYTE =

Byte-classification lookup tables (indexed 0..255). Scanning by raw byte via String#getbyte avoids allocating a one-character String and running the regex engine for every source character, which dominated lexing.

Array.new(256) { |b| b.chr.match?(/[A-Za-z_]/) }.freeze
IDENT_PART_BYTE = 
Array.new(256) { |b| b.chr.match?(/[A-Za-z0-9_]/) }.freeze
DIGIT_BYTE = 
Array.new(256) { |b| b.chr.match?(/[0-9]/) }.freeze
NUMERIC_PART_BYTE = 
Array.new(256) { |b| b.chr.match?(/[0-9_]/) }.freeze
HEX_DIGIT_BYTE = 
Array.new(256) { |b| b.chr.match?(/[0-9a-fA-F_]/) }.freeze
BIN_DIGIT_BYTE = 
Array.new(256) { |b| b.chr.match?(/[01_]/) }.freeze
SPACE_BYTE = 
" ".ord

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, path: nil, mode: :syntax_only, recovery_errors: nil) ⇒ Lexer

Returns a new instance of Lexer.



126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/milk_tea/core/lexer.rb', line 126

def initialize(source, path: nil, mode: :syntax_only, recovery_errors: nil)
  @source = source.gsub(/\r\n?/, "\n")
  @path = path
  @mode = mode
  @recovery_errors = recovery_errors
  @tokens = []
  @trivia = []
  @pending_leading_trivia = []
  @indent_stack = [0]
  @grouping_depth = 0
  @grouping_start_line = 0
  @grouping_start_column = 0
  @line_count = @source.empty? ? 1 : @source.lines.count
  @continuation_pending = false
end

Class Method Details

.lex(source, path: nil, mode: :syntax_only, recovery_errors: nil) ⇒ Object 



117
118
119
120
 
# File 'lib/milk_tea/core/lexer.rb', line 117

def self.lex(source, path: nil, mode: :syntax_only, recovery_errors: nil)
  result = new(source, path: path, mode:, recovery_errors:).lex
  mode == :with_trivia ? result.tokens : result
end

.lex_with_trivia(source, path: nil) ⇒ Object 



122
123
124
 
# File 'lib/milk_tea/core/lexer.rb', line 122

def self.lex_with_trivia(source, path: nil)
  new(source, path: path, mode: :with_trivia).lex
end

Instance Method Details

#lex ⇒ Object 



142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/milk_tea/core/lexer.rb', line 142

def lex
  lines = @source.each_line.to_a
  line_offset = 0
  line_number = 1
  line_index = 0

  while line_index < lines.length
    raw_line = lines[line_index]
    has_newline = raw_line.end_with?("\n")
    line = raw_line.delete_suffix("\n").b
    consumed_lines = lex_line(lines, line_index, line, line_number, line_offset, has_newline:)

    consumed_lines.times do |delta|
      line_offset += lines.fetch(line_index + delta).bytesize
    end
    line_number += consumed_lines
    line_index += consumed_lines
  end

  if @grouping_depth.positive?
    if @recovery_errors
      @recovery_errors << LexError.new("unclosed grouping delimiter", line: @grouping_start_line, column: @grouping_start_column, path: @path)
      @grouping_depth = 0
    else
      raise LexError.new("unclosed grouping delimiter", line: @line_count, column: 1, path: @path)
    end
  end

  while @indent_stack.length > 1
    @indent_stack.pop
    @tokens << token(:dedent, "", nil, @line_count, 1, start_offset: @source.bytesize, end_offset: @source.bytesize)
  end

  @tokens << token(:eof, "", nil, @line_count + 1, 1, start_offset: @source.bytesize, end_offset: @source.bytesize)
  return LexResult.new(tokens: @tokens, trivia: @trivia) if with_trivia?

  @tokens
end