Class: MilkTea::Parser

Inherits:
Object
  • Object
show all
Includes:
Attributes, Blocks, Declarations, Expressions, Recovery, Statements, Types
Defined in:
lib/milk_tea/core/parser.rb,
lib/milk_tea/core/parser/types.rb,
lib/milk_tea/core/parser/blocks.rb,
lib/milk_tea/core/parser/recovery.rb,
lib/milk_tea/core/parser/attributes.rb,
lib/milk_tea/core/parser/statements.rb,
lib/milk_tea/core/parser/expressions.rb,
lib/milk_tea/core/parser/declarations.rb

Defined Under Namespace

Modules: Attributes, Blocks, Declarations, Expressions, Recovery, Statements, Types Classes: ParseContext, ParseRecoveryResult

Constant Summary collapse

BUILTIN_ATTRIBUTE_NAME_LEXEMES = 
%w[packed align].freeze

Constants included from Declarations

Declarations::DECL_KIND

Constants included from Types

Types::BUILTIN_ATTRIBUTE_HANDLE_TYPE, Types::BUILTIN_CALLABLE_HANDLE_TYPE, Types::BUILTIN_FIELD_HANDLE_TYPE, Types::BUILTIN_IVECTOR_ELEMENT, Types::BUILTIN_MEMBER_HANDLE_TYPE, Types::BUILTIN_OPTION_TYPE, Types::BUILTIN_PRIMITIVE_NAMES, Types::BUILTIN_RESULT_TYPE, Types::BUILTIN_STRUCT_HANDLE_TYPE, Types::BUILTIN_TYPE_META_TYPE, Types::BUILTIN_TYPE_NAMES, Types::BUILTIN_VECTOR_ELEMENT, Types::RESERVED_IMPORT_ALIAS_NAMES, Types::RESERVED_TYPE_BINDING_NAMES, Types::RESERVED_VALUE_TYPE_NAMES

Constants included from Recovery

Recovery::TOP_LEVEL_RECOVERY_START_TYPES

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Statements

#check_inline_stmt_start?, #check_parallel_block_start?, #check_parallel_for_start?, #check_when_start?, #inline_block_body?, #match_arm_expression_form?, #normalize_mixed_match_arms, #parse_assignment_or_expression_stmt, #parse_break_stmt, #parse_continue_stmt, #parse_decl_when_arms, #parse_defer_stmt, #parse_destructure_pattern, #parse_else_branch_body, #parse_emit_stmt, #parse_for_stmt, #parse_gather_stmt, #parse_if_branch, #parse_if_stmt, #parse_inline_for_stmt, #parse_inline_if_stmt, #parse_inline_match_stmt, #parse_inline_or_block_body, #parse_inline_stmt, #parse_inline_while_stmt, #parse_local_decl, #parse_match_arm, #parse_match_arm_body, #parse_match_arms, #parse_match_stmt, #parse_parallel_block_stmt, #parse_parallel_for_stmt, #parse_pass_stmt, #parse_return_stmt, #parse_statement, #parse_static_assert, #parse_unsafe_stmt, #parse_when_decl, #parse_when_stmt, #parse_while_stmt, #reject_operator_statement_start!

Methods included from Declarations

#check_method_start?, #dispatch_decl_kind, #extending_target_type_param_names, #extending_target_type_param_names_from_argument, #parse_attribute_decl, #parse_block_body_with_recovery, #parse_callable_signature, #parse_compiler_flag_directive, #parse_const_decl, #parse_const_or_function_decl, #parse_declaration, #parse_enum_decl, #parse_event_decl, #parse_extending_block, #parse_extern_decl, #parse_extern_function_decl, #parse_foreign_decl, #parse_foreign_function_decl, #parse_function_def, #parse_import, #parse_include_directive, #parse_interface_decl, #parse_interface_method_decl, #parse_link_directive, #parse_method_def, #parse_method_head, #parse_method_kind, #parse_opaque_decl, #parse_optional_c_name, #parse_raw_module_body, #parse_raw_module_declaration, #parse_raw_module_directive, #parse_struct_decl, #parse_struct_decl_params, #parse_struct_member, #parse_type_alias_decl, #parse_union_decl, #parse_var_decl, #parse_variant_decl, #parse_visibility, #raw_module_declaration_error_message, #raw_module_directive_start?

Methods included from Expressions

#adjacent_tokens?, #aggregate_specialization_target?, #builtin_specialization_target?, #definite_type_argument?, #diagnostic_hint_error?, #explicit_specialization_argument?, #generic_callable_specialization_target?, #imported_member_specialization_target?, #parse_additive, #parse_adjacent_string_literal, #parse_alignof_expression, #parse_and, #parse_bitwise_and, #parse_bitwise_or, #parse_bitwise_xor, #parse_call_argument, #parse_call_arguments, #parse_comparison, #parse_embedded_expression, #parse_equality, #parse_expression, #parse_format_spec, #parse_format_string_literal, #parse_format_string_part, #parse_if_expression, #parse_is, #parse_left_associative, #parse_match_arm_pattern, #parse_match_expression, #parse_match_expression_arm, #parse_match_expression_arms, #parse_multiplicative, #parse_not, #parse_offsetof_expression, #parse_or, #parse_postfix, #parse_primary, #parse_proc_expression, #parse_range, #parse_shift, #parse_sizeof_expression, #parse_unary, #parse_unsafe_expression, #postfix_bracket_starts_specialization?, #potential_named_literal_type_argument?, #specialization_call_target?, #specialization_target?, #specialization_value_target?, #try_parse_prefix_cast_expression, #try_parse_specialization

Methods included from Attributes

#consume_attribute_name_component, #parse_attribute_application, #parse_attribute_applications, #parse_attribute_name, #parse_struct_layout_attributes, #reject_attributes!

Methods included from Types

array_type?, integer_type?, pointer_to, str_buffer_type?, substitute_type_variables

Methods included from Recovery

#recover_match_arm_block, #recover_statement_block_body, #recovery_error_block_stmt, #recovery_error_expr, #recovery_error_stmt, #recovery_statement_header_type, #synchronize_to_match_arm_boundary, #synchronize_to_statement_boundary, #synchronize_to_top_level_boundary, #top_level_recovery_start?

Methods included from Blocks

#parse_and_dedent_block_body, #parse_block, #parse_block_body, #parse_declaration_block, #parse_recoverable_block, #parse_statement_block_body, #unexpected_statement_block_indent_token

Constructor Details

#initialize(tokens, path: nil) ⇒ Parser

Returns a new instance of Parser.



97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/milk_tea/core/parser.rb', line 97

def initialize(tokens, path: nil)
  @tokens = tokens.is_a?(SyntaxTokenStream) ? tokens : SyntaxTokenStream.new(tokens)
  @path = path
  @current = 0
  @known_type_names = {}
  @known_import_aliases = {}
  @known_generic_callable_names = {}
  @current_type_param_names = []
  @in_inline_block_body = false
  seed_known_names
end

Class Method Details

.parse(source = nil, path: nil, tokens: nil) ⇒ Object 



81
82
83
84
 
# File 'lib/milk_tea/core/parser.rb', line 81

def self.parse(source = nil, path: nil, tokens: nil)
  token_stream = tokens || Lexer.lex(source, path: path)
  new(token_stream, path: path).parse
end

.parse_collecting_errors(source = nil, path: nil, tokens: nil) ⇒ Object 



86
87
88
89
90
91
92
93
94
95
 
# File 'lib/milk_tea/core/parser.rb', line 86

def self.parse_collecting_errors(source = nil, path: nil, tokens: nil)
  if tokens
    return new(tokens, path: path).parse_collecting_errors
  end

  lex_errors = []
  token_stream = Lexer.lex(source, path: path, recovery_errors: lex_errors)
  parse_result = new(token_stream, path: path).parse_collecting_errors
  ParseRecoveryResult.new(ast: parse_result.ast, errors: lex_errors + parse_result.errors)
end

Instance Method Details

#advance ⇒ Object 



200
201
202
203
 
# File 'lib/milk_tea/core/parser.rb', line 200

def advance
  @current += 1 unless eof?
  previous
end

#apply_parser_context(context) ⇒ Object 



366
367
368
369
370
371
 
# File 'lib/milk_tea/core/parser.rb', line 366

def apply_parser_context(context)
  @known_type_names = context.known_type_names
  @known_import_aliases = context.known_import_aliases
  @known_generic_callable_names = context.known_generic_callable_names
  @current_type_param_names = context.current_type_param_names
end

#at_external_file? ⇒ Boolean

Returns:

  • (Boolean) 


193
194
195
196
197
198
 
# File 'lib/milk_tea/core/parser.rb', line 193

def at_external_file?
  return false unless check(:external)

  next_token = @tokens[@current + 1]
  next_token.nil? || %i[newline eof].include?(next_token.type)
end

#block_expression?(expression) ⇒ Boolean

Returns:

  • (Boolean) 


299
300
301
 
# File 'lib/milk_tea/core/parser.rb', line 299

def block_expression?(expression)
  expression.is_a?(AST::ProcExpr) || expression.is_a?(AST::MatchExpr) || expression.is_a?(AST::IfExpr)
end

#builtin_attribute_identifier?(token) ⇒ Boolean

Returns:

  • (Boolean) 


337
338
339
 
# File 'lib/milk_tea/core/parser.rb', line 337

def builtin_attribute_identifier?(token)
  token&.type == :identifier && BUILTIN_ATTRIBUTE_NAME_LEXEMES.include?(token.lexeme)
end

#check(type) ⇒ Object 



258
259
260
261
262
 
# File 'lib/milk_tea/core/parser.rb', line 258

def check(type)
  return false if eof?

  peek.type == type
end

#check_name ⇒ Object 



264
265
266
 
# File 'lib/milk_tea/core/parser.rb', line 264

def check_name
  !eof? && peek.type == :identifier
end

#check_next(type) ⇒ Object 



268
269
270
271
272
 
# File 'lib/milk_tea/core/parser.rb', line 268

def check_next(type)
  return false if (@current + 1) >= @tokens.length

  @tokens[@current + 1].type == type
end

#consume(type, message) ⇒ Object 



224
225
226
227
228
 
# File 'lib/milk_tea/core/parser.rb', line 224

def consume(type, message)
  return advance if check(type)

  raise error(%i[rparen rbracket rbrace].include?(type) ? previous : peek, message)
end

#consume_end_of_statement ⇒ Object 



289
290
291
292
293
 
# File 'lib/milk_tea/core/parser.rb', line 289

def consume_end_of_statement
  return if @in_inline_block_body

  consume(:newline, "expected end of statement")
end

#consume_name(message) ⇒ Object 



230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/milk_tea/core/parser.rb', line 230

def consume_name(message)
  if keyword?(peek)
    name_role = message.sub(/\A(?:expected|required)\s+/, "")
    clearer_message = if name_role == message
                        message
                      else
                        "keyword '#{peek.lexeme}' cannot be used as #{name_role}"
                      end
    raise error(peek, clearer_message)
  end

  consume(:identifier, message)
end

#consume_name_allowing_keywords(message) ⇒ Object 



244
245
246
247
248
249
250
251
252
 
# File 'lib/milk_tea/core/parser.rb', line 244

def consume_name_allowing_keywords(message)
  if check(:identifier)
    advance
  elsif keyword?(peek)
    advance
  else
    raise error(peek, message)
  end
end

#consume_path_component(message) ⇒ Object 



311
312
313
314
315
 
# File 'lib/milk_tea/core/parser.rb', line 311

def consume_path_component(message)
  return advance if !eof? && (peek.type == :identifier || Token::KEYWORDS.value?(peek.type))

  raise error(peek, message)
end

#eof? ⇒ Boolean

Returns:

  • (Boolean) 


205
206
207
 
# File 'lib/milk_tea/core/parser.rb', line 205

def eof?
  peek.type == :eof
end

#error(token, message) ⇒ Object 



281
282
283
 
# File 'lib/milk_tea/core/parser.rb', line 281

def error(token, message)
  ParseError.new(message, token:, path: @path)
end

#finish_expression_statement(value) ⇒ Object 



295
296
297
 
# File 'lib/milk_tea/core/parser.rb', line 295

def finish_expression_statement(value)
  consume_end_of_statement unless block_expression?(value)
end

#foreign_param_mode_start? ⇒ Boolean

Returns:

  • (Boolean) 


331
332
333
334
335
 
# File 'lib/milk_tea/core/parser.rb', line 331

def foreign_param_mode_start?
  return false unless %i[out in inout consuming].include?(peek.type)

  @tokens[@current + 1]&.type == :identifier
end

#identifier_token?(token) ⇒ Boolean

Returns:

  • (Boolean) 


442
443
444
 
# File 'lib/milk_tea/core/parser.rb', line 442

def identifier_token?(token)
  token&.type == :identifier
end

#keyword?(token) ⇒ Boolean

Returns:

  • (Boolean) 


254
255
256
 
# File 'lib/milk_tea/core/parser.rb', line 254

def keyword?(token)
  token && Token::KEYWORDS.key?(token.lexeme)
end

#known_type_context_name?(name) ⇒ Boolean

Returns:

  • (Boolean) 


341
342
343
 
# File 'lib/milk_tea/core/parser.rb', line 341

def known_type_context_name?(name)
  @known_type_names.key?(name) || @known_import_aliases.key?(name) || @current_type_param_names.include?(name)
end

#match(*types) ⇒ Object 



217
218
219
220
221
222
 
# File 'lib/milk_tea/core/parser.rb', line 217

def match(*types)
  return false unless types.any? { |type| check(type) }

  advance
  true
end

#match_name ⇒ Object 



274
275
276
277
278
279
 
# File 'lib/milk_tea/core/parser.rb', line 274

def match_name
  return false unless check_name

  advance
  true
end

#matching_rbracket_index(start_index) ⇒ Object 



446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
 
# File 'lib/milk_tea/core/parser.rb', line 446

def matching_rbracket_index(start_index)
  depth = 0
  index = start_index

  while index < @tokens.length
    case @tokens[index].type
    when :lbracket
      depth += 1
    when :rbracket
      depth -= 1
      return index if depth.zero?
    end
    index += 1
  end

  nil
end

#parse ⇒ Object 



109
110
111
 
# File 'lib/milk_tea/core/parser.rb', line 109

def parse
  parse_source_file
end

#parse_collecting_errors ⇒ Object 



113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/milk_tea/core/parser.rb', line 113

def parse_collecting_errors
  errors = @recovery_errors ? @recovery_errors.dup : []
  previous_recovery_errors = @recovery_errors
  @recovery_errors = errors
  ast = parse_source_file(errors:)
  ParseRecoveryResult.new(ast:, errors:)
rescue ParseError => e
  errors ||= []
  errors << e
  ParseRecoveryResult.new(ast: nil, errors:)
ensure
  @recovery_errors = previous_recovery_errors
end

#parse_comma_separated_until(closing_type) ⇒ Object 



317
318
319
320
321
322
323
324
325
326
327
328
329
 
# File 'lib/milk_tea/core/parser.rb', line 317

def parse_comma_separated_until(closing_type)
  items = []

  unless check(closing_type)
    loop do
      items << yield
      break unless match(:comma)
      break if check(closing_type)
    end
  end

  items
end

#parse_qualified_name ⇒ Object 



303
304
305
306
307
308
309
 
# File 'lib/milk_tea/core/parser.rb', line 303

def parse_qualified_name
  parts = [consume_path_component("expected identifier").lexeme]
  while match(:dot)
    parts << consume_path_component("expected identifier after '.'").lexeme
  end
  AST::QualifiedName.new(parts:)
end

#parse_source_file(errors: nil) ⇒ Object 



129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
 
# File 'lib/milk_tea/core/parser.rb', line 129

def parse_source_file(errors: nil)
  skip_newlines

  module_name = nil
  module_kind = :module
  module_line = nil
  imports = []
  directives = []
  declarations = []

  if at_external_file?
    advance
    module_line = previous.line
    module_kind = :raw_module
    consume(:newline, "expected newline after external") unless eof?
    skip_newlines
    imports, directives, declarations = parse_raw_module_body(errors:)
    skip_newlines
    raise error(peek, "expected end of file after external declarations") unless eof?

    return AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
  end

  while match(:import)
    if errors
      begin
        imports << parse_import
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      imports << parse_import
    end
    skip_newlines
  end

  until eof?
    if errors
      begin
        result = parse_declaration
        if result.is_a?(Array)
          declarations.concat(result)
        else
          declarations << result
        end
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      result = parse_declaration
      if result.is_a?(Array)
        declarations.concat(result)
      else
        declarations << result
      end
    end
    skip_newlines
  end

  AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
end

#parser_context ⇒ Object

Returns a ParseContext snapshot of the parser state that must propagate to nested parsers (e.g. format string interpolations). When adding parser state that must be cloned, update both this method, apply_parser_context, and the ParseContext Data class.



357
358
359
360
361
362
363
364
 
# File 'lib/milk_tea/core/parser.rb', line 357

def parser_context
  ParseContext.new(
    known_type_names: @known_type_names.dup,
    known_import_aliases: @known_import_aliases.dup,
    known_generic_callable_names: @known_generic_callable_names.dup,
    current_type_param_names: @current_type_param_names.dup,
  )
end

#peek ⇒ Object 



209
210
211
 
# File 'lib/milk_tea/core/parser.rb', line 209

def peek
  @tokens[@current]
end

#previous ⇒ Object 



213
214
215
 
# File 'lib/milk_tea/core/parser.rb', line 213

def previous
  @tokens[@current - 1]
end

#seed_import_alias(start_index) ⇒ Object 



422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
 
# File 'lib/milk_tea/core/parser.rb', line 422

def seed_import_alias(start_index)
  cursor = start_index
  last_part = nil

  while cursor < @tokens.length && @tokens[cursor].type != :newline
    token = @tokens[cursor]
    if token.type == :as
      alias_token = @tokens[cursor + 1]
      @known_import_aliases[alias_token.lexeme] = true if identifier_token?(alias_token)
      return cursor
    end

    last_part = token.lexeme if identifier_token?(token)
    cursor += 1
  end

  @known_import_aliases[last_part] = true if last_part
  cursor
end

#seed_known_names ⇒ Object 



373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
 
# File 'lib/milk_tea/core/parser.rb', line 373

def seed_known_names
  MilkTea::BUILTIN_TYPE_NAMES.each { |name| @known_type_names[name] = true }

  depth = 0
  index = 0
  while index < @tokens.length
    token = @tokens[index]
    case token.type
    when :indent
      depth += 1
    when :dedent
      depth -= 1 if depth.positive?
    when :import
      index = seed_import_alias(index + 1) if depth.zero?
    when :function
      if depth.zero?
        name_token = @tokens[index + 1]
        type_param_token = @tokens[index + 2]
        if identifier_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :async
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if identifier_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :foreign
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if identifier_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :struct, :union, :enum, :flags, :opaque, :type, :variant
      if depth.zero?
        name_token = @tokens[index + 1]
        @known_type_names[name_token.lexeme] = true if identifier_token?(name_token)
      end
    end

    index += 1
  end
end

#skip_newlines ⇒ Object 



285
286
287
 
# File 'lib/milk_tea/core/parser.rb', line 285

def skip_newlines
  advance while check(:newline)
end

#with_type_param_names(names) ⇒ Object 



345
346
347
348
349
350
351
 
# File 'lib/milk_tea/core/parser.rb', line 345

def with_type_param_names(names)
  saved_names = @current_type_param_names
  @current_type_param_names = @current_type_param_names + names
  yield
ensure
  @current_type_param_names = saved_names
end