Class: MilkTea::Parser

Inherits:
Object
  • Object
show all
Includes:
Attributes, Blocks, Declarations, Expressions, Recovery, Statements, Types
Defined in:
lib/milk_tea/core/parser.rb,
lib/milk_tea/core/parser/types.rb,
lib/milk_tea/core/parser/blocks.rb,
lib/milk_tea/core/parser/recovery.rb,
lib/milk_tea/core/parser/attributes.rb,
lib/milk_tea/core/parser/statements.rb,
lib/milk_tea/core/parser/expressions.rb,
lib/milk_tea/core/parser/declarations.rb

Defined Under Namespace

Modules: Attributes, Blocks, Declarations, Expressions, Recovery, Statements, Types Classes: ParseContext, ParseRecoveryResult

Constant Summary collapse

BUILTIN_ATTRIBUTE_NAME_LEXEMES = 
%w[packed align].freeze

Constants included from Declarations

Declarations::DECL_KIND

Constants included from Types

Types::BUILTIN_ATTRIBUTE_HANDLE_TYPE, Types::BUILTIN_CALLABLE_HANDLE_TYPE, Types::BUILTIN_FIELD_HANDLE_TYPE, Types::BUILTIN_IVECTOR_ELEMENT, Types::BUILTIN_MEMBER_HANDLE_TYPE, Types::BUILTIN_OPTION_TYPE, Types::BUILTIN_PRIMITIVE_NAMES, Types::BUILTIN_RESULT_TYPE, Types::BUILTIN_STRUCT_HANDLE_TYPE, Types::BUILTIN_TYPE_META_TYPE, Types::BUILTIN_TYPE_NAMES, Types::BUILTIN_VECTOR_ELEMENT, Types::RESERVED_IMPORT_ALIAS_NAMES, Types::RESERVED_TYPE_BINDING_NAMES, Types::RESERVED_VALUE_TYPE_NAMES

Constants included from Recovery

Recovery::TOP_LEVEL_RECOVERY_START_TYPES

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Statements

#check_inline_stmt_start?, #check_parallel_block_start?, #check_parallel_for_start?, #check_when_start?, #inline_block_body?, #match_arm_expression_form?, #parse_assignment_or_expression_stmt, #parse_break_stmt, #parse_continue_stmt, #parse_decl_when_arms, #parse_defer_stmt, #parse_destructure_pattern, #parse_else_branch_body, #parse_emit_stmt, #parse_for_stmt, #parse_gather_stmt, #parse_if_branch, #parse_if_stmt, #parse_inline_for_stmt, #parse_inline_if_stmt, #parse_inline_match_stmt, #parse_inline_or_block_body, #parse_inline_stmt, #parse_inline_while_stmt, #parse_local_decl, #parse_match_arm, #parse_match_arm_body, #parse_match_arms, #parse_match_stmt, #parse_parallel_block_stmt, #parse_parallel_for_stmt, #parse_pass_stmt, #parse_return_stmt, #parse_statement, #parse_static_assert, #parse_unsafe_stmt, #parse_when_decl, #parse_when_stmt, #parse_while_stmt, #reject_operator_statement_start!

Methods included from Declarations

#check_method_start?, #dispatch_decl_kind, #extending_target_type_param_names, #extending_target_type_param_names_from_argument, #parse_attribute_decl, #parse_block_body_with_recovery, #parse_callable_signature, #parse_compiler_flag_directive, #parse_const_decl, #parse_const_or_function_decl, #parse_declaration, #parse_enum_decl, #parse_event_decl, #parse_extending_block, #parse_extern_decl, #parse_extern_function_decl, #parse_foreign_decl, #parse_foreign_function_decl, #parse_function_def, #parse_import, #parse_include_directive, #parse_interface_decl, #parse_interface_method_decl, #parse_link_directive, #parse_method_def, #parse_method_head, #parse_method_kind, #parse_opaque_decl, #parse_optional_c_name, #parse_raw_module_body, #parse_raw_module_declaration, #parse_raw_module_directive, #parse_struct_decl, #parse_struct_decl_params, #parse_struct_member, #parse_type_alias_decl, #parse_union_decl, #parse_var_decl, #parse_variant_decl, #parse_visibility, #raw_module_declaration_error_message, #raw_module_directive_start?

Methods included from Expressions

#adjacent_tokens?, #aggregate_specialization_target?, #builtin_specialization_target?, #definite_type_argument?, #diagnostic_hint_error?, #explicit_specialization_argument?, #generic_callable_specialization_target?, #imported_member_specialization_target?, #parse_additive, #parse_adjacent_string_literal, #parse_alignof_expression, #parse_and, #parse_bitwise_and, #parse_bitwise_or, #parse_bitwise_xor, #parse_call_argument, #parse_call_arguments, #parse_comparison, #parse_embedded_expression, #parse_equality, #parse_expression, #parse_format_spec, #parse_format_string_literal, #parse_format_string_part, #parse_if_expression, #parse_is, #parse_left_associative, #parse_match_arm_pattern, #parse_match_expression, #parse_match_expression_arm, #parse_match_expression_arms, #parse_multiplicative, #parse_not, #parse_offsetof_expression, #parse_or, #parse_postfix, #parse_primary, #parse_proc_expression, #parse_range, #parse_shift, #parse_sizeof_expression, #parse_unary, #parse_unsafe_expression, #postfix_bracket_starts_specialization?, #potential_named_literal_type_argument?, #specialization_call_target?, #specialization_target?, #specialization_value_target?, #try_parse_prefix_cast_expression, #try_parse_specialization

Methods included from Attributes

#consume_attribute_name_component, #parse_attribute_application, #parse_attribute_applications, #parse_attribute_name, #parse_struct_layout_attributes, #reject_attributes!

Methods included from Types

array_type?, integer_type?, pointer_to, str_buffer_type?, substitute_type_variables

Methods included from Recovery

#recover_match_arm_block, #recover_statement_block_body, #recovery_error_block_stmt, #recovery_error_expr, #recovery_error_stmt, #recovery_statement_header_type, #synchronize_to_match_arm_boundary, #synchronize_to_statement_boundary, #synchronize_to_top_level_boundary, #top_level_recovery_start?

Methods included from Blocks

#parse_and_dedent_block_body, #parse_block, #parse_block_body, #parse_declaration_block, #parse_recoverable_block, #parse_statement_block_body, #unexpected_statement_block_indent_token

Constructor Details

#initialize(tokens, path: nil) ⇒ Parser

Returns a new instance of Parser.



85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/milk_tea/core/parser.rb', line 85

def initialize(tokens, path: nil)
  @tokens = tokens.is_a?(SyntaxTokenStream) ? tokens : SyntaxTokenStream.new(tokens)
  @path = path
  @current = 0
  @known_type_names = {}
  @known_import_aliases = {}
  @known_generic_callable_names = {}
  @current_type_param_names = []
  @in_inline_block_body = false
  seed_known_names
end

Class Method Details

.parse(source = nil, path: nil, tokens: nil) ⇒ Object 



69
70
71
72
 
# File 'lib/milk_tea/core/parser.rb', line 69

def self.parse(source = nil, path: nil, tokens: nil)
  token_stream = tokens || Lexer.lex(source, path: path)
  new(token_stream, path: path).parse
end

.parse_collecting_errors(source = nil, path: nil, tokens: nil) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
 
# File 'lib/milk_tea/core/parser.rb', line 74

def self.parse_collecting_errors(source = nil, path: nil, tokens: nil)
  if tokens
    return new(tokens, path: path).parse_collecting_errors
  end

  lex_errors = []
  token_stream = Lexer.lex(source, path: path, recovery_errors: lex_errors)
  parse_result = new(token_stream, path: path).parse_collecting_errors
  ParseRecoveryResult.new(ast: parse_result.ast, errors: lex_errors + parse_result.errors)
end

Instance Method Details

#advance ⇒ Object 



188
189
190
191
 
# File 'lib/milk_tea/core/parser.rb', line 188

def advance
  @current += 1 unless eof?
  previous
end

#apply_parser_context(context) ⇒ Object 



354
355
356
357
358
359
 
# File 'lib/milk_tea/core/parser.rb', line 354

def apply_parser_context(context)
  @known_type_names = context.known_type_names
  @known_import_aliases = context.known_import_aliases
  @known_generic_callable_names = context.known_generic_callable_names
  @current_type_param_names = context.current_type_param_names
end

#at_external_file? ⇒ Boolean

Returns:

  • (Boolean) 


181
182
183
184
185
186
 
# File 'lib/milk_tea/core/parser.rb', line 181

def at_external_file?
  return false unless check(:external)

  next_token = @tokens[@current + 1]
  next_token.nil? || %i[newline eof].include?(next_token.type)
end

#block_expression?(expression) ⇒ Boolean

Returns:

  • (Boolean) 


287
288
289
 
# File 'lib/milk_tea/core/parser.rb', line 287

def block_expression?(expression)
  expression.is_a?(AST::ProcExpr) || expression.is_a?(AST::MatchExpr) || expression.is_a?(AST::IfExpr)
end

#builtin_attribute_identifier?(token) ⇒ Boolean

Returns:

  • (Boolean) 


325
326
327
 
# File 'lib/milk_tea/core/parser.rb', line 325

def builtin_attribute_identifier?(token)
  token&.type == :identifier && BUILTIN_ATTRIBUTE_NAME_LEXEMES.include?(token.lexeme)
end

#check(type) ⇒ Object 



246
247
248
249
250
 
# File 'lib/milk_tea/core/parser.rb', line 246

def check(type)
  return false if eof?

  peek.type == type
end

#check_name ⇒ Object 



252
253
254
 
# File 'lib/milk_tea/core/parser.rb', line 252

def check_name
  !eof? && peek.type == :identifier
end

#check_next(type) ⇒ Object 



256
257
258
259
260
 
# File 'lib/milk_tea/core/parser.rb', line 256

def check_next(type)
  return false if (@current + 1) >= @tokens.length

  @tokens[@current + 1].type == type
end

#consume(type, message) ⇒ Object 



212
213
214
215
216
 
# File 'lib/milk_tea/core/parser.rb', line 212

def consume(type, message)
  return advance if check(type)

  raise error(%i[rparen rbracket rbrace].include?(type) ? previous : peek, message)
end

#consume_end_of_statement ⇒ Object 



277
278
279
280
281
 
# File 'lib/milk_tea/core/parser.rb', line 277

def consume_end_of_statement
  return if @in_inline_block_body

  consume(:newline, "expected end of statement")
end

#consume_name(message) ⇒ Object 



218
219
220
221
222
223
224
225
226
227
228
229
230
 
# File 'lib/milk_tea/core/parser.rb', line 218

def consume_name(message)
  if keyword?(peek)
    name_role = message.sub(/\A(?:expected|required)\s+/, "")
    clearer_message = if name_role == message
                        message
                      else
                        "keyword '#{peek.lexeme}' cannot be used as #{name_role}"
                      end
    raise error(peek, clearer_message)
  end

  consume(:identifier, message)
end

#consume_name_allowing_keywords(message) ⇒ Object 



232
233
234
235
236
237
238
239
240
 
# File 'lib/milk_tea/core/parser.rb', line 232

def consume_name_allowing_keywords(message)
  if check(:identifier)
    advance
  elsif keyword?(peek)
    advance
  else
    raise error(peek, message)
  end
end

#consume_path_component(message) ⇒ Object 



299
300
301
302
303
 
# File 'lib/milk_tea/core/parser.rb', line 299

def consume_path_component(message)
  return advance if !eof? && (peek.type == :identifier || Token::KEYWORDS.value?(peek.type))

  raise error(peek, message)
end

#eof? ⇒ Boolean

Returns:

  • (Boolean) 


193
194
195
 
# File 'lib/milk_tea/core/parser.rb', line 193

def eof?
  peek.type == :eof
end

#error(token, message) ⇒ Object 



269
270
271
 
# File 'lib/milk_tea/core/parser.rb', line 269

def error(token, message)
  ParseError.new(message, token:, path: @path)
end

#finish_expression_statement(value) ⇒ Object 



283
284
285
 
# File 'lib/milk_tea/core/parser.rb', line 283

def finish_expression_statement(value)
  consume_end_of_statement unless block_expression?(value)
end

#foreign_param_mode_start? ⇒ Boolean

Returns:

  • (Boolean) 


319
320
321
322
323
 
# File 'lib/milk_tea/core/parser.rb', line 319

def foreign_param_mode_start?
  return false unless %i[out in inout consuming].include?(peek.type)

  @tokens[@current + 1]&.type == :identifier
end

#identifier_token?(token) ⇒ Boolean

Returns:

  • (Boolean) 


430
431
432
 
# File 'lib/milk_tea/core/parser.rb', line 430

def identifier_token?(token)
  token&.type == :identifier
end

#keyword?(token) ⇒ Boolean

Returns:

  • (Boolean) 


242
243
244
 
# File 'lib/milk_tea/core/parser.rb', line 242

def keyword?(token)
  token && Token::KEYWORDS.key?(token.lexeme)
end

#known_type_context_name?(name) ⇒ Boolean

Returns:

  • (Boolean) 


329
330
331
 
# File 'lib/milk_tea/core/parser.rb', line 329

def known_type_context_name?(name)
  @known_type_names.key?(name) || @known_import_aliases.key?(name) || @current_type_param_names.include?(name)
end

#match(*types) ⇒ Object 



205
206
207
208
209
210
 
# File 'lib/milk_tea/core/parser.rb', line 205

def match(*types)
  return false unless types.any? { |type| check(type) }

  advance
  true
end

#match_name ⇒ Object 



262
263
264
265
266
267
 
# File 'lib/milk_tea/core/parser.rb', line 262

def match_name
  return false unless check_name

  advance
  true
end

#matching_rbracket_index(start_index) ⇒ Object 



434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
 
# File 'lib/milk_tea/core/parser.rb', line 434

def matching_rbracket_index(start_index)
  depth = 0
  index = start_index

  while index < @tokens.length
    case @tokens[index].type
    when :lbracket
      depth += 1
    when :rbracket
      depth -= 1
      return index if depth.zero?
    end
    index += 1
  end

  nil
end

#parse ⇒ Object 



97
98
99
 
# File 'lib/milk_tea/core/parser.rb', line 97

def parse
  parse_source_file
end

#parse_collecting_errors ⇒ Object 



101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/milk_tea/core/parser.rb', line 101

def parse_collecting_errors
  errors = @recovery_errors ? @recovery_errors.dup : []
  previous_recovery_errors = @recovery_errors
  @recovery_errors = errors
  ast = parse_source_file(errors:)
  ParseRecoveryResult.new(ast:, errors:)
rescue ParseError => e
  errors ||= []
  errors << e
  ParseRecoveryResult.new(ast: nil, errors:)
ensure
  @recovery_errors = previous_recovery_errors
end

#parse_comma_separated_until(closing_type) ⇒ Object 



305
306
307
308
309
310
311
312
313
314
315
316
317
 
# File 'lib/milk_tea/core/parser.rb', line 305

def parse_comma_separated_until(closing_type)
  items = []

  unless check(closing_type)
    loop do
      items << yield
      break unless match(:comma)
      break if check(closing_type)
    end
  end

  items
end

#parse_qualified_name ⇒ Object 



291
292
293
294
295
296
297
 
# File 'lib/milk_tea/core/parser.rb', line 291

def parse_qualified_name
  parts = [consume_path_component("expected identifier").lexeme]
  while match(:dot)
    parts << consume_path_component("expected identifier after '.'").lexeme
  end
  AST::QualifiedName.new(parts:)
end

#parse_source_file(errors: nil) ⇒ Object 



117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/milk_tea/core/parser.rb', line 117

def parse_source_file(errors: nil)
  skip_newlines

  module_name = nil
  module_kind = :module
  module_line = nil
  imports = []
  directives = []
  declarations = []

  if at_external_file?
    advance
    module_line = previous.line
    module_kind = :raw_module
    consume(:newline, "expected newline after external") unless eof?
    skip_newlines
    imports, directives, declarations = parse_raw_module_body(errors:)
    skip_newlines
    raise error(peek, "expected end of file after external declarations") unless eof?

    return AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
  end

  while match(:import)
    if errors
      begin
        imports << parse_import
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      imports << parse_import
    end
    skip_newlines
  end

  until eof?
    if errors
      begin
        result = parse_declaration
        if result.is_a?(Array)
          declarations.concat(result)
        else
          declarations << result
        end
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      result = parse_declaration
      if result.is_a?(Array)
        declarations.concat(result)
      else
        declarations << result
      end
    end
    skip_newlines
  end

  AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
end

#parser_context ⇒ Object

Returns a ParseContext snapshot of the parser state that must propagate to nested parsers (e.g. format string interpolations). When adding parser state that must be cloned, update both this method, apply_parser_context, and the ParseContext Data class.



345
346
347
348
349
350
351
352
 
# File 'lib/milk_tea/core/parser.rb', line 345

def parser_context
  ParseContext.new(
    known_type_names: @known_type_names.dup,
    known_import_aliases: @known_import_aliases.dup,
    known_generic_callable_names: @known_generic_callable_names.dup,
    current_type_param_names: @current_type_param_names.dup,
  )
end

#peek ⇒ Object 



197
198
199
 
# File 'lib/milk_tea/core/parser.rb', line 197

def peek
  @tokens[@current]
end

#previous ⇒ Object 



201
202
203
 
# File 'lib/milk_tea/core/parser.rb', line 201

def previous
  @tokens[@current - 1]
end

#seed_import_alias(start_index) ⇒ Object 



410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
 
# File 'lib/milk_tea/core/parser.rb', line 410

def seed_import_alias(start_index)
  cursor = start_index
  last_part = nil

  while cursor < @tokens.length && @tokens[cursor].type != :newline
    token = @tokens[cursor]
    if token.type == :as
      alias_token = @tokens[cursor + 1]
      @known_import_aliases[alias_token.lexeme] = true if identifier_token?(alias_token)
      return cursor
    end

    last_part = token.lexeme if identifier_token?(token)
    cursor += 1
  end

  @known_import_aliases[last_part] = true if last_part
  cursor
end

#seed_known_names ⇒ Object 



361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
 
# File 'lib/milk_tea/core/parser.rb', line 361

def seed_known_names
  MilkTea::BUILTIN_TYPE_NAMES.each { |name| @known_type_names[name] = true }

  depth = 0
  index = 0
  while index < @tokens.length
    token = @tokens[index]
    case token.type
    when :indent
      depth += 1
    when :dedent
      depth -= 1 if depth.positive?
    when :import
      index = seed_import_alias(index + 1) if depth.zero?
    when :function
      if depth.zero?
        name_token = @tokens[index + 1]
        type_param_token = @tokens[index + 2]
        if identifier_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :async
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if identifier_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :foreign
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if identifier_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :struct, :union, :enum, :flags, :opaque, :type, :variant
      if depth.zero?
        name_token = @tokens[index + 1]
        @known_type_names[name_token.lexeme] = true if identifier_token?(name_token)
      end
    end

    index += 1
  end
end

#skip_newlines ⇒ Object 



273
274
275
 
# File 'lib/milk_tea/core/parser.rb', line 273

def skip_newlines
  advance while check(:newline)
end

#with_type_param_names(names) ⇒ Object 



333
334
335
336
337
338
339
 
# File 'lib/milk_tea/core/parser.rb', line 333

def with_type_param_names(names)
  saved_names = @current_type_param_names
  @current_type_param_names = @current_type_param_names + names
  yield
ensure
  @current_type_param_names = saved_names
end