Class: MilkTea::Lexer

Inherits:
Object
  • Object
show all
Includes:
CharacterClasses, FormatStrings, Heredocs, Indentation, Numbers, Recovery, Strings, Symbols, Trivia
Defined in:
lib/milk_tea/core/lexer.rb,
lib/milk_tea/core/lexer/trivia.rb,
lib/milk_tea/core/lexer/numbers.rb,
lib/milk_tea/core/lexer/strings.rb,
lib/milk_tea/core/lexer/symbols.rb,
lib/milk_tea/core/lexer/heredocs.rb,
lib/milk_tea/core/lexer/recovery.rb,
lib/milk_tea/core/lexer/indentation.rb,
lib/milk_tea/core/lexer/format_strings.rb,
lib/milk_tea/core/lexer/character_classes.rb

Defined Under Namespace

Modules: CharacterClasses, FormatStrings, Heredocs, Indentation, Numbers, Recovery, Strings, Symbols, Trivia Classes: LexResult, StringLexResult, StringSegment

Constant Summary collapse

TOP_LEVEL_RESYNC_PREFIXES = 
%w[
  attribute const enum external flags foreign function include import interface
  link opaque public static_assert struct type union var variant extending event
].freeze
LINE_CONTINUATION_OPERATORS = 
Token::LINE_CONTINUATION_OPERATORS
THREE_CHAR_TOKENS = 
{
  "..." => :ellipsis,
  "<<=" => :shift_left_equal,
  ">>=" => :shift_right_equal,
}.freeze
TWO_CHAR_TOKENS = 
{
  "->" => :arrow,
  ".." => :dot_dot,
  "<<" => :shift_left,
  ">>" => :shift_right,
  "+=" => :plus_equal,
  "-=" => :minus_equal,
  "*=" => :star_equal,
  "/=" => :slash_equal,
  "%=" => :percent_equal,
  "&=" => :amp_equal,
  "|=" => :pipe_equal,
  "^=" => :caret_equal,
  "==" => :equal_equal,
  "!=" => :bang_equal,
  "<=" => :less_equal,
  ">=" => :greater_equal,
}.freeze
ONE_CHAR_TOKENS = 
{
  "&" => :amp,
  "@" => :at,
  ":" => :colon,
  "," => :comma,
  "^" => :caret,
  "." => :dot,
  "(" => :lparen,
  ")" => :rparen,
  "|" => :pipe,
  "[" => :lbracket,
  "]" => :rbracket,
  "?" => :question,
  "=" => :equal,
  "+" => :plus,
  "-" => :minus,
  "*" => :star,
  "/" => :slash,
  "%" => :percent,
  "<" => :less,
  ">" => :greater,
  "~" => :tilde,
}.freeze
INTEGER_SUFFIX_STRINGS = 
%w[ub us ul iz b s i u l z].sort_by { |s| -s.length }.freeze
IDENT_START_BYTE =

Byte-classification lookup tables (indexed 0..255). Scanning by raw byte via String#getbyte avoids allocating a one-character String and running the regex engine for every source character, which dominated lexing.

Array.new(256) { |b| b.chr.match?(/[A-Za-z_]/) }.freeze
IDENT_PART_BYTE = 
Array.new(256) { |b| b.chr.match?(/[A-Za-z0-9_]/) }.freeze
DIGIT_BYTE = 
Array.new(256) { |b| b.chr.match?(/[0-9]/) }.freeze
NUMERIC_PART_BYTE = 
Array.new(256) { |b| b.chr.match?(/[0-9_]/) }.freeze
HEX_DIGIT_BYTE = 
Array.new(256) { |b| b.chr.match?(/[0-9a-fA-F_]/) }.freeze
BIN_DIGIT_BYTE = 
Array.new(256) { |b| b.chr.match?(/[01_]/) }.freeze
SPACE_BYTE = 
" ".ord

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Recovery

#top_level_resync_line?

Methods included from Symbols

#adjust_grouping_depth, #lex_symbol

Methods included from FormatStrings

#advance_heredoc_position, #format_spec_suffix?, #lex_format_string, #parse_format_heredoc_parts, #scan_format_interpolation_end, #skip_interpolation_string_contents, #skip_string_contents, #split_format_interpolation_source

Methods included from Heredocs

#dedent_heredoc_content, #dedent_heredoc_lines, #heredoc_content_margin, #heredoc_context_allowed?, #heredoc_prefix, #heredoc_start?, #heredoc_terminator?, #lex_heredoc

Methods included from Strings

#decode_escape, #lex_char_literal, #lex_string, #scan_string_segment

Methods included from Numbers

#exponent_part?, #lex_number, #parse_integer, #scan_integer_suffix_at

Methods included from Indentation

#emit_indentation, #recover_indentation

Methods included from Trivia

#append_trailing_or_pending, #push_pending_leading_trivia, #register_detached_line_trivia, #with_trivia?

Methods included from CharacterClasses

#digit?, #identifier_part?, #identifier_start?, #identifier_start_token, #leading_space_count, #lex_identifier

Constructor Details

#initialize(source, path: nil, mode: :syntax_only, recovery_errors: nil) ⇒ Lexer

Returns a new instance of Lexer.



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/milk_tea/core/lexer.rb', line 118

def initialize(source, path: nil, mode: :syntax_only, recovery_errors: nil)
  @source = source.gsub(/\r\n?/, "\n")
  @path = path
  @mode = mode
  @recovery_errors = recovery_errors
  @tokens = []
  @trivia = []
  @pending_leading_trivia = []
  @indent_stack = [0]
  @grouping_depth = 0
  @grouping_start_line = 0
  @grouping_start_column = 0
  @line_count = @source.empty? ? 1 : @source.lines.count
  @continuation_pending = false
end

Class Method Details

.lex(source, path: nil, mode: :syntax_only, recovery_errors: nil) ⇒ Object 



109
110
111
112
 
# File 'lib/milk_tea/core/lexer.rb', line 109

def self.lex(source, path: nil, mode: :syntax_only, recovery_errors: nil)
  result = new(source, path: path, mode:, recovery_errors:).lex
  mode == :with_trivia ? result.tokens : result
end

.lex_with_trivia(source, path: nil) ⇒ Object 



114
115
116
 
# File 'lib/milk_tea/core/lexer.rb', line 114

def self.lex_with_trivia(source, path: nil)
  new(source, path: path, mode: :with_trivia).lex
end

Instance Method Details

#consume_pending_leading_trivia ⇒ Object 



412
413
414
415
416
417
418
 
# File 'lib/milk_tea/core/lexer.rb', line 412

def consume_pending_leading_trivia
  return [] unless with_trivia?

  trivia = @pending_leading_trivia
  @pending_leading_trivia = []
  trivia
end

#emit_line_newline(line, line_number, line_offset, has_newline) ⇒ Object

── newline emission ────────────────────────────────────────────



376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
 
# File 'lib/milk_tea/core/lexer.rb', line 376

def emit_line_newline(line, line_number, line_offset, has_newline)
  newline_start = line_offset + line.bytesize
  newline_end = has_newline ? (newline_start + 1) : newline_start

  if @grouping_depth.zero?
    @tokens << token(:newline, "\n", nil, line_number, line.bytesize + 1, start_offset: newline_start, end_offset: newline_end)
  elsif with_trivia? && has_newline
    append_trailing_or_pending(
      TriviaToken.new(
        kind: :newline,
        text: "\n",
        line: line_number,
        column: line.bytesize + 1,
        start_offset: newline_start,
        end_offset: newline_end,
      ),
    )
  end
end

#lex ⇒ Object 



134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/milk_tea/core/lexer.rb', line 134

def lex
  lines = @source.each_line.to_a
  line_offset = 0
  line_number = 1
  line_index = 0

  while line_index < lines.length
    raw_line = lines[line_index]
    has_newline = raw_line.end_with?("\n")
    line = raw_line.delete_suffix("\n").b
    consumed_lines = lex_line(lines, line_index, line, line_number, line_offset, has_newline:)

    consumed_lines.times do |delta|
      line_offset += lines.fetch(line_index + delta).bytesize
    end
    line_number += consumed_lines
    line_index += consumed_lines
  end

  if @grouping_depth.positive?
    if @recovery_errors
      @recovery_errors << LexError.new("unclosed grouping delimiter", line: @grouping_start_line, column: @grouping_start_column, path: @path)
      @grouping_depth = 0
    else
      raise LexError.new("unclosed grouping delimiter", line: @line_count, column: 1, path: @path)
    end
  end

  while @indent_stack.length > 1
    @indent_stack.pop
    @tokens << token(:dedent, "", nil, @line_count, 1, start_offset: @source.bytesize, end_offset: @source.bytesize)
  end

  @tokens << token(:eof, "", nil, @line_count + 1, 1, start_offset: @source.bytesize, end_offset: @source.bytesize)
  return LexResult.new(tokens: @tokens, trivia: @trivia) if with_trivia?

  @tokens
end

#lex_line(lines, line_index, line, line_number, line_offset, has_newline:) ⇒ Object 



183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
 
# File 'lib/milk_tea/core/lexer.rb', line 183

def lex_line(lines, line_index, line, line_number, line_offset, has_newline:)
  tab_index = line.index("\t")
  if tab_index
    error = LexError.new("tabs are not allowed; use 4 spaces for indentation", line: line_number, column: tab_index + 1, path: @path)
    if @recovery_errors
      @recovery_errors << error
      line = line.gsub("\t", "    ")
    else
      raise error
    end
  end

  if line.strip.empty?
    register_detached_line_trivia(:blank_line, line, line_number, line_offset, has_newline:)
    return 1
  end

  if line.lstrip.start_with?("#")
    register_detached_line_trivia(:comment, line, line_number, line_offset, has_newline:)
    return 1
  end

  index = leading_space_count(line)
  if @recovery_errors && @grouping_depth.positive? && index.zero? && top_level_resync_line?(line)
    @recovery_errors << LexError.new("unclosed grouping delimiter", line: @grouping_start_line, column: @grouping_start_column, path: @path)
    @grouping_depth = 0
    @tokens << token(:newline, "\n", nil, line_number, 1, start_offset: line_offset, end_offset: line_offset)
  end

  if with_trivia? && index.positive?
    push_pending_leading_trivia(
      TriviaToken.new(
        kind: :space,
        text: line[0...index],
        line: line_number,
        column: 1,
        start_offset: line_offset,
        end_offset: line_offset + index,
      ),
    )
  end

  if @grouping_depth.zero?
    if @continuation_pending
    elsif @recovery_errors
      begin
        emit_indentation(index, line_number, line_offset)
      rescue LexError => e
        @recovery_errors << e
        recover_indentation(index, line_number, line_offset)
      end
    else
      emit_indentation(index, line_number, line_offset)
    end
  end
  @continuation_pending = false

  while index < line.length
    char = line[index]

    if char == " "
      if with_trivia?
        span_start = index
        index += 1 while index < line.length && line[index] == " "
        push_pending_leading_trivia(
          TriviaToken.new(
            kind: :space,
            text: line[span_start...index],
            line: line_number,
            column: span_start + 1,
            start_offset: line_offset + span_start,
            end_offset: line_offset + index,
          ),
        )
        next
      end

      index += 1
      next
    end

    if char == "#"
      if with_trivia?
        comment_end = line.length
        comment_text = line[index...comment_end]
        append_trailing_or_pending(
          TriviaToken.new(
            kind: :comment,
            text: comment_text,
            line: line_number,
            column: index + 1,
            start_offset: line_offset + index,
            end_offset: line_offset + comment_end,
          ),
        )
      end
      break
    end

    if char == "c" && heredoc_start?(line, index, cstring: true)
      return lex_heredoc(lines, line_index, index, line_number, line_offset, cstring: true)
    end

    if char == "c" && line[index + 1] == "<" && line[index + 2] == "-" && identifier_start?(line[index + 3])
      error = LexError.new("expected '<<-' for heredoc string; did you mean 'c<<-#{identifier_start_token(line, index + 3)}'?", line: line_number, column: index + 1, path: @path)
      if @recovery_errors
        @recovery_errors << error
      else
        raise error
      end
    end
    if char == "c" && line[index + 1] == '"'
      result = lex_string(lines, line_index, line, index, line_number, line_offset:, cstring: true)
      return result.consumed_lines if result.consumed_lines > 1

      index = result.next_index
      next
    end

    if char == "f" && heredoc_start?(line, index, format: true)
      return lex_heredoc(lines, line_index, index, line_number, line_offset, cstring: false, format: true)
    end

    if char == "f" && line[index + 1] == "<" && line[index + 2] == "-" && identifier_start?(line[index + 3])
      error = LexError.new("expected '<<-' for heredoc string; did you mean 'f<<-#{identifier_start_token(line, index + 3)}'?", line: line_number, column: index + 1, path: @path)
      if @recovery_errors
        @recovery_errors << error
      else
        raise error
      end
    end

    if char == "f" && line[index + 1] == '"'
      index = lex_format_string(line, index, line_number, line_offset:)
      next
    end

    if char == "<" && heredoc_start?(line, index)
      return lex_heredoc(lines, line_index, index, line_number, line_offset, cstring: false)
    end

    if char == '"'
      result = lex_string(lines, line_index, line, index, line_number, line_offset:)
      return result.consumed_lines if result.consumed_lines > 1

      index = result.next_index
      next
    end

    if char == "'"
      index = lex_char_literal(line, index, line_number, line_offset:)
      next
    end

    if identifier_start?(char)
      index = lex_identifier(line, index, line_number, line_offset:)
      next
    end

    if digit?(char)
      index = lex_number(line, index, line_number, line_offset:)
      next
    end

    index = lex_symbol(line, index, line_number, line_offset:)
  end

  newline_start = line_offset + line.length
  newline_end = has_newline ? (newline_start + 1) : newline_start
  if @grouping_depth.zero?
    if Token::LINE_CONTINUATION_OPERATORS.include?(@tokens.last&.type)
      @continuation_pending = true
    else
      @tokens << token(:newline, "\n", nil, line_number, line.length + 1, start_offset: newline_start, end_offset: newline_end)
    end
  elsif with_trivia? && has_newline
    append_trailing_or_pending(
      TriviaToken.new(
        kind: :newline,
        text: "\n",
        line: line_number,
        column: line.length + 1,
        start_offset: newline_start,
        end_offset: newline_end,
      ),
    )
  end

  1
end

#token(type, lexeme, literal, line, column, start_offset:, end_offset:) ⇒ Object

── token construction ──────────────────────────────────────────



398
399
400
401
402
403
404
405
406
407
408
409
410
 
# File 'lib/milk_tea/core/lexer.rb', line 398

def token(type, lexeme, literal, line, column, start_offset:, end_offset:)
  Token.new(
    type:,
    lexeme:,
    literal:,
    line:,
    column:,
    start_offset:,
    end_offset:,
    leading_trivia: consume_pending_leading_trivia,
    trailing_trivia: [],
  )
end