Class: MilkTea::Parser

Inherits:
Object
  • Object
show all
Includes:
Attributes, Blocks, Declarations, Expressions, Recovery, Statements, Types
Defined in:
lib/milk_tea/core/parser.rb,
lib/milk_tea/core/parser/types.rb,
lib/milk_tea/core/parser/blocks.rb,
lib/milk_tea/core/parser/recovery.rb,
lib/milk_tea/core/parser/attributes.rb,
lib/milk_tea/core/parser/statements.rb,
lib/milk_tea/core/parser/expressions.rb,
lib/milk_tea/core/parser/declarations.rb

Defined Under Namespace

Modules: Attributes, Blocks, Declarations, Expressions, Recovery, Statements, Types Classes: ParseContext, ParseRecoveryResult

Constant Summary collapse

BUILTIN_ATTRIBUTE_NAME_LEXEMES = 
%w[packed align].freeze

Constants included from Declarations

Declarations::DECL_KIND

Constants included from Types

Types::BUILTIN_ATTRIBUTE_HANDLE_TYPE, Types::BUILTIN_CALLABLE_HANDLE_TYPE, Types::BUILTIN_FIELD_HANDLE_TYPE, Types::BUILTIN_IVECTOR_ELEMENT, Types::BUILTIN_MEMBER_HANDLE_TYPE, Types::BUILTIN_OPTION_TYPE, Types::BUILTIN_PRIMITIVE_NAMES, Types::BUILTIN_RESULT_TYPE, Types::BUILTIN_STRUCT_HANDLE_TYPE, Types::BUILTIN_TYPE_META_TYPE, Types::BUILTIN_TYPE_NAMES, Types::BUILTIN_VECTOR_ELEMENT, Types::RESERVED_IMPORT_ALIAS_NAMES, Types::RESERVED_TYPE_BINDING_NAMES, Types::RESERVED_VALUE_TYPE_NAMES

Constants included from Recovery

Recovery::TOP_LEVEL_RECOVERY_START_TYPES

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Statements

#check_inline_stmt_start?, #check_parallel_block_start?, #check_parallel_for_start?, #check_when_start?, #inline_block_body?, #match_arm_expression_form?, #parse_assignment_or_expression_stmt, #parse_break_stmt, #parse_continue_stmt, #parse_decl_when_arms, #parse_defer_stmt, #parse_destructure_pattern, #parse_else_branch_body, #parse_emit_stmt, #parse_for_stmt, #parse_gather_stmt, #parse_if_branch, #parse_if_stmt, #parse_inline_for_stmt, #parse_inline_if_stmt, #parse_inline_match_stmt, #parse_inline_or_block_body, #parse_inline_stmt, #parse_inline_while_stmt, #parse_local_decl, #parse_match_arm, #parse_match_arm_body, #parse_match_arms, #parse_match_stmt, #parse_parallel_block_stmt, #parse_parallel_for_stmt, #parse_pass_stmt, #parse_return_stmt, #parse_statement, #parse_static_assert, #parse_unsafe_stmt, #parse_when_decl, #parse_when_stmt, #parse_while_stmt, #reject_operator_statement_start!

Methods included from Declarations

#dispatch_decl_kind, #extending_target_type_param_names, #extending_target_type_param_names_from_argument, #parse_attribute_decl, #parse_block_body_with_recovery, #parse_callable_signature, #parse_compiler_flag_directive, #parse_const_decl, #parse_const_or_function_decl, #parse_declaration, #parse_enum_decl, #parse_event_decl, #parse_extending_block, #parse_extern_decl, #parse_extern_function_decl, #parse_foreign_decl, #parse_foreign_function_decl, #parse_function_def, #parse_import, #parse_include_directive, #parse_interface_decl, #parse_interface_method_decl, #parse_link_directive, #parse_method_def, #parse_method_head, #parse_method_kind, #parse_opaque_decl, #parse_optional_c_name, #parse_raw_module_body, #parse_raw_module_declaration, #parse_raw_module_directive, #parse_struct_decl, #parse_struct_decl_params, #parse_struct_member, #parse_type_alias_decl, #parse_union_decl, #parse_var_decl, #parse_variant_decl, #parse_visibility, #raw_module_declaration_error_message, #raw_module_directive_start?

Methods included from Expressions

#adjacent_tokens?, #aggregate_specialization_target?, #builtin_specialization_target?, #definite_type_argument?, #diagnostic_hint_error?, #explicit_specialization_argument?, #generic_callable_specialization_target?, #imported_member_specialization_target?, #parse_additive, #parse_adjacent_string_literal, #parse_alignof_expression, #parse_and, #parse_bitwise_and, #parse_bitwise_or, #parse_bitwise_xor, #parse_call_argument, #parse_call_arguments, #parse_comparison, #parse_embedded_expression, #parse_equality, #parse_expression, #parse_format_spec, #parse_format_string_literal, #parse_format_string_part, #parse_if_expression, #parse_is, #parse_left_associative, #parse_match_arm_pattern, #parse_match_expression, #parse_match_expression_arm, #parse_match_expression_arms, #parse_multiplicative, #parse_not, #parse_offsetof_expression, #parse_or, #parse_postfix, #parse_primary, #parse_proc_expression, #parse_range, #parse_shift, #parse_sizeof_expression, #parse_unary, #parse_unsafe_expression, #postfix_bracket_starts_specialization?, #potential_named_literal_type_argument?, #specialization_call_target?, #specialization_target?, #specialization_value_target?, #try_parse_prefix_cast_expression, #try_parse_specialization

Methods included from Attributes

#consume_attribute_name_component, #parse_attribute_application, #parse_attribute_applications, #parse_attribute_name, #parse_struct_layout_attributes, #reject_attributes!

Methods included from Types

array_type?, integer_type?, pointer_to, str_buffer_type?, substitute_type_variables

Methods included from Recovery

#recover_match_arm_block, #recover_statement_block_body, #recovery_error_block_stmt, #recovery_error_expr, #recovery_error_stmt, #recovery_statement_header_type, #synchronize_to_match_arm_boundary, #synchronize_to_statement_boundary, #synchronize_to_top_level_boundary, #top_level_recovery_start?

Methods included from Blocks

#parse_and_dedent_block_body, #parse_block, #parse_block_body, #parse_declaration_block, #parse_recoverable_block, #parse_statement_block_body, #unexpected_statement_block_indent_token

Constructor Details

#initialize(tokens, path: nil) ⇒ Parser

Returns a new instance of Parser.



85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/milk_tea/core/parser.rb', line 85

def initialize(tokens, path: nil)
  @tokens = tokens.is_a?(SyntaxTokenStream) ? tokens : SyntaxTokenStream.new(tokens)
  @path = path
  @current = 0
  @known_type_names = {}
  @known_import_aliases = {}
  @known_generic_callable_names = {}
  @current_type_param_names = []
  @in_inline_block_body = false
  seed_known_names
end

Class Method Details

.parse(source = nil, path: nil, tokens: nil) ⇒ Object 



69
70
71
72
 
# File 'lib/milk_tea/core/parser.rb', line 69

def self.parse(source = nil, path: nil, tokens: nil)
  token_stream = tokens || Lexer.lex(source, path: path)
  new(token_stream, path: path).parse
end

.parse_collecting_errors(source = nil, path: nil, tokens: nil) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
 
# File 'lib/milk_tea/core/parser.rb', line 74

def self.parse_collecting_errors(source = nil, path: nil, tokens: nil)
  if tokens
    return new(tokens, path: path).parse_collecting_errors
  end

  lex_errors = []
  token_stream = Lexer.lex(source, path: path, recovery_errors: lex_errors)
  parse_result = new(token_stream, path: path).parse_collecting_errors
  ParseRecoveryResult.new(ast: parse_result.ast, errors: lex_errors + parse_result.errors)
end

Instance Method Details

#advance ⇒ Object 



178
179
180
181
 
# File 'lib/milk_tea/core/parser.rb', line 178

def advance
  @current += 1 unless eof?
  previous
end

#apply_parser_context(context) ⇒ Object 



344
345
346
347
348
349
 
# File 'lib/milk_tea/core/parser.rb', line 344

def apply_parser_context(context)
  @known_type_names = context.known_type_names
  @known_import_aliases = context.known_import_aliases
  @known_generic_callable_names = context.known_generic_callable_names
  @current_type_param_names = context.current_type_param_names
end

#at_external_file? ⇒ Boolean

Returns:

  • (Boolean) 


171
172
173
174
175
176
 
# File 'lib/milk_tea/core/parser.rb', line 171

def at_external_file?
  return false unless check(:external)

  next_token = @tokens[@current + 1]
  next_token.nil? || %i[newline eof].include?(next_token.type)
end

#block_expression?(expression) ⇒ Boolean

Returns:

  • (Boolean) 


277
278
279
 
# File 'lib/milk_tea/core/parser.rb', line 277

def block_expression?(expression)
  expression.is_a?(AST::ProcExpr) || expression.is_a?(AST::MatchExpr) || expression.is_a?(AST::IfExpr)
end

#builtin_attribute_identifier?(token) ⇒ Boolean

Returns:

  • (Boolean) 


315
316
317
 
# File 'lib/milk_tea/core/parser.rb', line 315

def builtin_attribute_identifier?(token)
  token&.type == :identifier && BUILTIN_ATTRIBUTE_NAME_LEXEMES.include?(token.lexeme)
end

#check(type) ⇒ Object 



236
237
238
239
240
 
# File 'lib/milk_tea/core/parser.rb', line 236

def check(type)
  return false if eof?

  peek.type == type
end

#check_name ⇒ Object 



242
243
244
 
# File 'lib/milk_tea/core/parser.rb', line 242

def check_name
  !eof? && peek.type == :identifier
end

#check_next(type) ⇒ Object 



246
247
248
249
250
 
# File 'lib/milk_tea/core/parser.rb', line 246

def check_next(type)
  return false if (@current + 1) >= @tokens.length

  @tokens[@current + 1].type == type
end

#consume(type, message) ⇒ Object 



202
203
204
205
206
 
# File 'lib/milk_tea/core/parser.rb', line 202

def consume(type, message)
  return advance if check(type)

  raise error(%i[rparen rbracket rbrace].include?(type) ? previous : peek, message)
end

#consume_end_of_statement ⇒ Object 



267
268
269
270
271
 
# File 'lib/milk_tea/core/parser.rb', line 267

def consume_end_of_statement
  return if @in_inline_block_body

  consume(:newline, "expected end of statement")
end

#consume_name(message) ⇒ Object 



208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/milk_tea/core/parser.rb', line 208

def consume_name(message)
  if keyword?(peek)
    name_role = message.sub(/\A(?:expected|required)\s+/, "")
    clearer_message = if name_role == message
                        message
                      else
                        "keyword '#{peek.lexeme}' cannot be used as #{name_role}"
                      end
    raise error(peek, clearer_message)
  end

  consume(:identifier, message)
end

#consume_name_allowing_keywords(message) ⇒ Object 



222
223
224
225
226
227
228
229
230
 
# File 'lib/milk_tea/core/parser.rb', line 222

def consume_name_allowing_keywords(message)
  if check(:identifier)
    advance
  elsif keyword?(peek)
    advance
  else
    raise error(peek, message)
  end
end

#consume_path_component(message) ⇒ Object 



289
290
291
292
293
 
# File 'lib/milk_tea/core/parser.rb', line 289

def consume_path_component(message)
  return advance if !eof? && (peek.type == :identifier || Token::KEYWORDS.value?(peek.type))

  raise error(peek, message)
end

#eof? ⇒ Boolean

Returns:

  • (Boolean) 


183
184
185
 
# File 'lib/milk_tea/core/parser.rb', line 183

def eof?
  peek.type == :eof
end

#error(token, message) ⇒ Object 



259
260
261
 
# File 'lib/milk_tea/core/parser.rb', line 259

def error(token, message)
  ParseError.new(message, token:, path: @path)
end

#finish_expression_statement(value) ⇒ Object 



273
274
275
 
# File 'lib/milk_tea/core/parser.rb', line 273

def finish_expression_statement(value)
  consume_end_of_statement unless block_expression?(value)
end

#foreign_param_mode_start? ⇒ Boolean

Returns:

  • (Boolean) 


309
310
311
312
313
 
# File 'lib/milk_tea/core/parser.rb', line 309

def foreign_param_mode_start?
  return false unless %i[out in inout consuming].include?(peek.type)

  @tokens[@current + 1]&.type == :identifier
end

#identifier_token?(token) ⇒ Boolean

Returns:

  • (Boolean) 


420
421
422
 
# File 'lib/milk_tea/core/parser.rb', line 420

def identifier_token?(token)
  token&.type == :identifier
end

#keyword?(token) ⇒ Boolean

Returns:

  • (Boolean) 


232
233
234
 
# File 'lib/milk_tea/core/parser.rb', line 232

def keyword?(token)
  token && Token::KEYWORDS.key?(token.lexeme)
end

#known_type_context_name?(name) ⇒ Boolean

Returns:

  • (Boolean) 


319
320
321
 
# File 'lib/milk_tea/core/parser.rb', line 319

def known_type_context_name?(name)
  @known_type_names.key?(name) || @known_import_aliases.key?(name) || @current_type_param_names.include?(name)
end

#match(*types) ⇒ Object 



195
196
197
198
199
200
 
# File 'lib/milk_tea/core/parser.rb', line 195

def match(*types)
  return false unless types.any? { |type| check(type) }

  advance
  true
end

#match_name ⇒ Object 



252
253
254
255
256
257
 
# File 'lib/milk_tea/core/parser.rb', line 252

def match_name
  return false unless check_name

  advance
  true
end

#matching_rbracket_index(start_index) ⇒ Object 



424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
 
# File 'lib/milk_tea/core/parser.rb', line 424

def matching_rbracket_index(start_index)
  depth = 0
  index = start_index

  while index < @tokens.length
    case @tokens[index].type
    when :lbracket
      depth += 1
    when :rbracket
      depth -= 1
      return index if depth.zero?
    end
    index += 1
  end

  nil
end

#parse ⇒ Object 



97
98
99
 
# File 'lib/milk_tea/core/parser.rb', line 97

def parse
  parse_source_file
end

#parse_collecting_errors ⇒ Object 



101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/milk_tea/core/parser.rb', line 101

def parse_collecting_errors
  errors = @recovery_errors ? @recovery_errors.dup : []
  previous_recovery_errors = @recovery_errors
  @recovery_errors = errors
  ast = parse_source_file(errors:)
  ParseRecoveryResult.new(ast:, errors:)
rescue ParseError => e
  errors ||= []
  errors << e
  ParseRecoveryResult.new(ast: nil, errors:)
ensure
  @recovery_errors = previous_recovery_errors
end

#parse_comma_separated_until(closing_type) ⇒ Object 



295
296
297
298
299
300
301
302
303
304
305
306
307
 
# File 'lib/milk_tea/core/parser.rb', line 295

def parse_comma_separated_until(closing_type)
  items = []

  unless check(closing_type)
    loop do
      items << yield
      break unless match(:comma)
      break if check(closing_type)
    end
  end

  items
end

#parse_qualified_name ⇒ Object 



281
282
283
284
285
286
287
 
# File 'lib/milk_tea/core/parser.rb', line 281

def parse_qualified_name
  parts = [consume_path_component("expected identifier").lexeme]
  while match(:dot)
    parts << consume_path_component("expected identifier after '.'").lexeme
  end
  AST::QualifiedName.new(parts:)
end

#parse_source_file(errors: nil) ⇒ Object 



117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/milk_tea/core/parser.rb', line 117

def parse_source_file(errors: nil)
  skip_newlines

  module_name = nil
  module_kind = :module
  module_line = nil
  imports = []
  directives = []
  declarations = []

  if at_external_file?
    advance
    module_line = previous.line
    module_kind = :raw_module
    consume(:newline, "expected newline after external") unless eof?
    skip_newlines
    imports, directives, declarations = parse_raw_module_body(errors:)
    skip_newlines
    raise error(peek, "expected end of file after external declarations") unless eof?

    return AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
  end

  while match(:import)
    if errors
      begin
        imports << parse_import
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      imports << parse_import
    end
    skip_newlines
  end

  until eof?
    if errors
      begin
        declarations << parse_declaration
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      declarations << parse_declaration
    end
    skip_newlines
  end

  AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
end

#parser_context ⇒ Object

Returns a ParseContext snapshot of the parser state that must propagate to nested parsers (e.g. format string interpolations). When adding parser state that must be cloned, update both this method, apply_parser_context, and the ParseContext Data class.



335
336
337
338
339
340
341
342
 
# File 'lib/milk_tea/core/parser.rb', line 335

def parser_context
  ParseContext.new(
    known_type_names: @known_type_names.dup,
    known_import_aliases: @known_import_aliases.dup,
    known_generic_callable_names: @known_generic_callable_names.dup,
    current_type_param_names: @current_type_param_names.dup,
  )
end

#peek ⇒ Object 



187
188
189
 
# File 'lib/milk_tea/core/parser.rb', line 187

def peek
  @tokens[@current]
end

#previous ⇒ Object 



191
192
193
 
# File 'lib/milk_tea/core/parser.rb', line 191

def previous
  @tokens[@current - 1]
end

#seed_import_alias(start_index) ⇒ Object 



400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
 
# File 'lib/milk_tea/core/parser.rb', line 400

def seed_import_alias(start_index)
  cursor = start_index
  last_part = nil

  while cursor < @tokens.length && @tokens[cursor].type != :newline
    token = @tokens[cursor]
    if token.type == :as
      alias_token = @tokens[cursor + 1]
      @known_import_aliases[alias_token.lexeme] = true if identifier_token?(alias_token)
      return cursor
    end

    last_part = token.lexeme if identifier_token?(token)
    cursor += 1
  end

  @known_import_aliases[last_part] = true if last_part
  cursor
end

#seed_known_names ⇒ Object 



351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
 
# File 'lib/milk_tea/core/parser.rb', line 351

def seed_known_names
  MilkTea::BUILTIN_TYPE_NAMES.each { |name| @known_type_names[name] = true }

  depth = 0
  index = 0
  while index < @tokens.length
    token = @tokens[index]
    case token.type
    when :indent
      depth += 1
    when :dedent
      depth -= 1 if depth.positive?
    when :import
      index = seed_import_alias(index + 1) if depth.zero?
    when :function
      if depth.zero?
        name_token = @tokens[index + 1]
        type_param_token = @tokens[index + 2]
        if identifier_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :async
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if identifier_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :foreign
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if identifier_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :struct, :union, :enum, :flags, :opaque, :type, :variant
      if depth.zero?
        name_token = @tokens[index + 1]
        @known_type_names[name_token.lexeme] = true if identifier_token?(name_token)
      end
    end

    index += 1
  end
end

#skip_newlines ⇒ Object 



263
264
265
 
# File 'lib/milk_tea/core/parser.rb', line 263

def skip_newlines
  advance while check(:newline)
end

#with_type_param_names(names) ⇒ Object 



323
324
325
326
327
328
329
 
# File 'lib/milk_tea/core/parser.rb', line 323

def with_type_param_names(names)
  saved_names = @current_type_param_names
  @current_type_param_names = @current_type_param_names + names
  yield
ensure
  @current_type_param_names = saved_names
end