Class: MilkTea::Parser

Inherits:
Object
  • Object
show all
Includes:
MilkTea::Parse::Attributes, MilkTea::Parse::Blocks, MilkTea::Parse::Declarations, MilkTea::Parse::Expressions, MilkTea::Parse::Recovery, MilkTea::Parse::Statements, MilkTea::Parse::Types
Defined in:
lib/milk_tea/core/parser.rb

Defined Under Namespace

Classes: ParseContext, ParseRecoveryResult

Constant Summary collapse

BUILTIN_ATTRIBUTE_NAME_LEXEMES = 
%w[packed align].freeze

Constants included from MilkTea::Parse::Declarations

MilkTea::Parse::Declarations::DECL_KIND

Constants included from MilkTea::Parse::Recovery

MilkTea::Parse::Recovery::TOP_LEVEL_RECOVERY_START_TYPES

Class Method Summary collapse

Instance Method Summary collapse

Methods included from MilkTea::Parse::Statements

#check_inline_stmt_start?, #check_parallel_block_start?, #check_parallel_for_start?, #check_when_start?, #inline_block_body?, #match_arm_expr_form?, #parse_assignment_or_expression_stmt, #parse_break_stmt, #parse_continue_stmt, #parse_decl_when_arms, #parse_defer_stmt, #parse_destructure_pattern, #parse_else_branch_body, #parse_emit_stmt, #parse_for_stmt, #parse_gather_stmt, #parse_if_branch, #parse_if_stmt, #parse_inline_for_stmt, #parse_inline_if_stmt, #parse_inline_match_stmt, #parse_inline_or_block_body, #parse_inline_stmt, #parse_inline_while_stmt, #parse_local_decl, #parse_match_arm, #parse_match_arm_body, #parse_match_arms, #parse_match_stmt, #parse_parallel_block_stmt, #parse_parallel_for_stmt, #parse_pass_stmt, #parse_return_stmt, #parse_statement, #parse_static_assert, #parse_unsafe_stmt, #parse_when_decl, #parse_when_stmt, #parse_while_stmt, #reject_operator_statement_start!

Methods included from MilkTea::Parse::Declarations

#dispatch_decl_kind, #extending_target_type_param_names, #extending_target_type_param_names_from_argument, #parse_attribute_decl, #parse_block_body_safe, #parse_callable_signature, #parse_compiler_flag_directive, #parse_const_decl, #parse_const_or_function_decl, #parse_declaration, #parse_enum_decl, #parse_event_decl, #parse_extending_block, #parse_extern_decl, #parse_extern_function_decl, #parse_foreign_decl, #parse_foreign_function_decl, #parse_function_def, #parse_import, #parse_include_directive, #parse_interface_decl, #parse_interface_method_decl, #parse_link_directive, #parse_method_def, #parse_method_kind, #parse_method_like_decl_head, #parse_opaque_decl, #parse_optional_explicit_c_name, #parse_raw_module_body, #parse_raw_module_declaration, #parse_raw_module_directive, #parse_struct_decl, #parse_struct_decl_params, #parse_struct_member, #parse_type_alias_decl, #parse_union_decl, #parse_var_decl, #parse_variant_decl, #parse_visibility, #raw_module_declaration_error_message, #raw_module_directive_start?

Methods included from MilkTea::Parse::Expressions

#adjacent_tokens?, #aggregate_specialization_target?, #builtin_specialization_target?, #definite_type_argument?, #explicit_specialization_argument?, #generic_callable_specialization_target?, #imported_member_specialization_target?, #parse_additive, #parse_adjacent_string_literal, #parse_alignof_expr, #parse_and, #parse_bitwise_and, #parse_bitwise_or, #parse_bitwise_xor, #parse_call_argument, #parse_call_arguments, #parse_comparison, #parse_diagnostic_hint?, #parse_embedded_expression, #parse_equality, #parse_expression, #parse_format_spec, #parse_format_string_literal, #parse_format_string_part, #parse_if_expression, #parse_is, #parse_left_associative, #parse_match_arm_pattern, #parse_match_expression, #parse_match_expression_arm, #parse_match_expression_arms, #parse_multiplicative, #parse_not, #parse_offsetof_expr, #parse_or, #parse_postfix, #parse_primary, #parse_proc_expr, #parse_range, #parse_shift, #parse_sizeof_expr, #parse_unary, #parse_unsafe_expression, #postfix_bracket_starts_specialization?, #potential_named_literal_type_argument?, #specialization_call_target?, #specialization_target?, #specialization_value_target?, #try_parse_prefix_cast_expression, #try_parse_specialization

Methods included from MilkTea::Parse::Attributes

#consume_attribute_name_component, #parse_attribute_application, #parse_attribute_applications, #parse_attribute_name, #parse_struct_layout_attributes, #reject_attributes!

Methods included from MilkTea::Parse::Types

#parse_callable_type_ref, #parse_declaration_type_params, #parse_dyn_type_ref, #parse_foreign_param, #parse_foreign_params, #parse_function_type_param, #parse_function_type_ref, #parse_implements_clause, #parse_param, #parse_param_default_value, #parse_parameter_list, #parse_params, #parse_proc_type_ref, #parse_qualified_name_with_type_arguments, #parse_signature_params, #parse_tuple_type_ref, #parse_type_argument, #parse_type_param_constraint, #parse_type_param_constraints, #parse_type_ref

Methods included from MilkTea::Parse::Recovery

#recover_match_arm_block, #recover_statement_block_body, #recovery_error_block_stmt, #recovery_error_expr, #recovery_error_stmt, #recovery_statement_header_type, #synchronize_to_match_arm_boundary, #synchronize_to_statement_boundary, #synchronize_to_top_level_boundary, #top_level_recovery_start?

Methods included from MilkTea::Parse::Blocks

#parse_and_dedent_block_body, #parse_block, #parse_block_body, #parse_declaration_block, #parse_named_block, #parse_statement_block_body, #unexpected_statement_block_indent_token

Constructor Details

#initialize(tokens, path: nil) ⇒ Parser

Returns a new instance of Parser.



67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/milk_tea/core/parser.rb', line 67

def initialize(tokens, path: nil)
  @tokens = tokens.is_a?(SyntaxTokenStream) ? tokens : SyntaxTokenStream.new(tokens)
  @path = path
  @current = 0
  @known_type_names = {}
  @known_import_aliases = {}
  @known_generic_callable_names = {}
  @current_type_param_names = []
  @in_inline_block_body = false
  seed_known_names
end

Class Method Details

.parse(source = nil, path: nil, tokens: nil) ⇒ Object 



51
52
53
54
 
# File 'lib/milk_tea/core/parser.rb', line 51

def self.parse(source = nil, path: nil, tokens: nil)
  token_stream = tokens || Lexer.lex(source, path: path)
  new(token_stream, path: path).parse
end

.parse_collecting_errors(source = nil, path: nil, tokens: nil) ⇒ Object 



56
57
58
59
60
61
62
63
64
65
 
# File 'lib/milk_tea/core/parser.rb', line 56

def self.parse_collecting_errors(source = nil, path: nil, tokens: nil)
  if tokens
    return new(tokens, path: path).parse_collecting_errors
  end

  lex_errors = []
  token_stream = Lexer.lex(source, path: path, recovery_errors: lex_errors)
  parse_result = new(token_stream, path: path).parse_collecting_errors
  ParseRecoveryResult.new(ast: parse_result.ast, errors: lex_errors + parse_result.errors)
end

Instance Method Details

#advance ⇒ Object 



160
161
162
163
 
# File 'lib/milk_tea/core/parser.rb', line 160

def advance
  @current += 1 unless eof?
  previous
end

#apply_parser_context(context) ⇒ Object 



326
327
328
329
330
331
 
# File 'lib/milk_tea/core/parser.rb', line 326

def apply_parser_context(context)
  @known_type_names = context.known_type_names
  @known_import_aliases = context.known_import_aliases
  @known_generic_callable_names = context.known_generic_callable_names
  @current_type_param_names = context.current_type_param_names
end

#block_expression?(expression) ⇒ Boolean

Returns:

  • (Boolean) 


259
260
261
 
# File 'lib/milk_tea/core/parser.rb', line 259

def block_expression?(expression)
  expression.is_a?(AST::ProcExpr) || expression.is_a?(AST::MatchExpr) || expression.is_a?(AST::IfExpr)
end

#builtin_attribute_identifier?(token) ⇒ Boolean

Returns:

  • (Boolean) 


297
298
299
 
# File 'lib/milk_tea/core/parser.rb', line 297

def builtin_attribute_identifier?(token)
  token&.type == :identifier && BUILTIN_ATTRIBUTE_NAME_LEXEMES.include?(token.lexeme)
end

#check(type) ⇒ Object 



218
219
220
221
222
 
# File 'lib/milk_tea/core/parser.rb', line 218

def check(type)
  return false if eof?

  peek.type == type
end

#check_name ⇒ Object 



224
225
226
 
# File 'lib/milk_tea/core/parser.rb', line 224

def check_name
  !eof? && peek.type == :identifier
end

#check_next(type) ⇒ Object 



228
229
230
231
232
 
# File 'lib/milk_tea/core/parser.rb', line 228

def check_next(type)
  return false if (@current + 1) >= @tokens.length

  @tokens[@current + 1].type == type
end

#consume(type, message) ⇒ Object 



184
185
186
187
188
 
# File 'lib/milk_tea/core/parser.rb', line 184

def consume(type, message)
  return advance if check(type)

  raise error(%i[rparen rbracket rbrace].include?(type) ? previous : peek, message)
end

#consume_end_of_statement ⇒ Object 



249
250
251
252
253
 
# File 'lib/milk_tea/core/parser.rb', line 249

def consume_end_of_statement
  return if @in_inline_block_body

  consume(:newline, "expected end of statement")
end

#consume_name(message) ⇒ Object 



190
191
192
193
194
195
196
197
198
199
200
201
202
 
# File 'lib/milk_tea/core/parser.rb', line 190

def consume_name(message)
  if keyword_token?(peek)
    name_role = message.sub(/\A(?:expected|required)\s+/, "")
    clearer_message = if name_role == message
                        message
                      else
                        "keyword '#{peek.lexeme}' cannot be used as #{name_role}"
                      end
    raise error(peek, clearer_message)
  end

  consume(:identifier, message)
end

#consume_name_allowing_keywords(message) ⇒ Object 



204
205
206
207
208
209
210
211
212
 
# File 'lib/milk_tea/core/parser.rb', line 204

def consume_name_allowing_keywords(message)
  if check(:identifier)
    advance
  elsif keyword_token?(peek)
    advance
  else
    raise error(peek, message)
  end
end

#consume_path_component(message) ⇒ Object 



271
272
273
274
275
 
# File 'lib/milk_tea/core/parser.rb', line 271

def consume_path_component(message)
  return advance if !eof? && (peek.type == :identifier || Token::KEYWORDS.value?(peek.type))

  raise error(peek, message)
end

#eof? ⇒ Boolean

Returns:

  • (Boolean) 


165
166
167
 
# File 'lib/milk_tea/core/parser.rb', line 165

def eof?
  peek.type == :eof
end

#error(token, message) ⇒ Object 



241
242
243
 
# File 'lib/milk_tea/core/parser.rb', line 241

def error(token, message)
  ParseError.new(message, token:, path: @path)
end

#external_file_header? ⇒ Boolean

Returns:

  • (Boolean) 


153
154
155
156
157
158
 
# File 'lib/milk_tea/core/parser.rb', line 153

def external_file_header?
  return false unless check(:external)

  next_token = @tokens[@current + 1]
  next_token.nil? || %i[newline eof].include?(next_token.type)
end

#finish_expression_statement(value) ⇒ Object 



255
256
257
 
# File 'lib/milk_tea/core/parser.rb', line 255

def finish_expression_statement(value)
  consume_end_of_statement unless block_expression?(value)
end

#foreign_param_qualifier_mode? ⇒ Boolean

Returns:

  • (Boolean) 


291
292
293
294
295
 
# File 'lib/milk_tea/core/parser.rb', line 291

def foreign_param_qualifier_mode?
  return false unless %i[out in inout consuming].include?(peek.type)

  @tokens[@current + 1]&.type == :identifier
end

#keyword_token?(token) ⇒ Boolean

Returns:

  • (Boolean) 


214
215
216
 
# File 'lib/milk_tea/core/parser.rb', line 214

def keyword_token?(token)
  token && Token::KEYWORDS.key?(token.lexeme)
end

#known_type_like_name?(name) ⇒ Boolean

Returns:

  • (Boolean) 


301
302
303
 
# File 'lib/milk_tea/core/parser.rb', line 301

def known_type_like_name?(name)
  @known_type_names.key?(name) || @known_import_aliases.key?(name) || @current_type_param_names.include?(name)
end

#match(*types) ⇒ Object 



177
178
179
180
181
182
 
# File 'lib/milk_tea/core/parser.rb', line 177

def match(*types)
  return false unless types.any? { |type| check(type) }

  advance
  true
end

#match_name ⇒ Object 



234
235
236
237
238
239
 
# File 'lib/milk_tea/core/parser.rb', line 234

def match_name
  return false unless check_name

  advance
  true
end

#matching_rbracket_index(start_index) ⇒ Object 



406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
 
# File 'lib/milk_tea/core/parser.rb', line 406

def matching_rbracket_index(start_index)
  depth = 0
  index = start_index

  while index < @tokens.length
    case @tokens[index].type
    when :lbracket
      depth += 1
    when :rbracket
      depth -= 1
      return index if depth.zero?
    end
    index += 1
  end

  nil
end

#parse ⇒ Object 



79
80
81
 
# File 'lib/milk_tea/core/parser.rb', line 79

def parse
  parse_source_file
end

#parse_collecting_errors ⇒ Object 



83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/milk_tea/core/parser.rb', line 83

def parse_collecting_errors
  errors = @recovery_errors ? @recovery_errors.dup : []
  previous_recovery_errors = @recovery_errors
  @recovery_errors = errors
  ast = parse_source_file(errors:)
  ParseRecoveryResult.new(ast:, errors:)
rescue ParseError => e
  errors ||= []
  errors << e
  ParseRecoveryResult.new(ast: nil, errors:)
ensure
  @recovery_errors = previous_recovery_errors
end

#parse_comma_separated_until(closing_type) ⇒ Object 



277
278
279
280
281
282
283
284
285
286
287
288
289
 
# File 'lib/milk_tea/core/parser.rb', line 277

def parse_comma_separated_until(closing_type)
  items = []

  unless check(closing_type)
    loop do
      items << yield
      break unless match(:comma)
      break if check(closing_type)
    end
  end

  items
end

#parse_qualified_name ⇒ Object 



263
264
265
266
267
268
269
 
# File 'lib/milk_tea/core/parser.rb', line 263

def parse_qualified_name
  parts = [consume_path_component("expected identifier").lexeme]
  while match(:dot)
    parts << consume_path_component("expected identifier after '.'").lexeme
  end
  AST::QualifiedName.new(parts:)
end

#parse_source_file(errors: nil) ⇒ Object 



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
 
# File 'lib/milk_tea/core/parser.rb', line 99

def parse_source_file(errors: nil)
  skip_newlines

  module_name = nil
  module_kind = :module
  module_line = nil
  imports = []
  directives = []
  declarations = []

  if external_file_header?
    advance
    module_line = previous.line
    module_kind = :raw_module
    consume(:newline, "expected newline after external") unless eof?
    skip_newlines
    imports, directives, declarations = parse_raw_module_body(errors:)
    skip_newlines
    raise error(peek, "expected end of file after external declarations") unless eof?

    return AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
  end

  while match(:import)
    if errors
      begin
        imports << parse_import
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      imports << parse_import
    end
    skip_newlines
  end

  until eof?
    if errors
      begin
        declarations << parse_declaration
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      declarations << parse_declaration
    end
    skip_newlines
  end

  AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
end

#parser_context ⇒ Object

Returns a ParseContext snapshot of the parser state that must propagate to nested parsers (e.g. format string interpolations). When adding parser state that must be cloned, update both this method, apply_parser_context, and the ParseContext Data class.



317
318
319
320
321
322
323
324
 
# File 'lib/milk_tea/core/parser.rb', line 317

def parser_context
  ParseContext.new(
    known_type_names: @known_type_names.dup,
    known_import_aliases: @known_import_aliases.dup,
    known_generic_callable_names: @known_generic_callable_names.dup,
    current_type_param_names: @current_type_param_names.dup,
  )
end

#peek ⇒ Object 



169
170
171
 
# File 'lib/milk_tea/core/parser.rb', line 169

def peek
  @tokens[@current]
end

#previous ⇒ Object 



173
174
175
 
# File 'lib/milk_tea/core/parser.rb', line 173

def previous
  @tokens[@current - 1]
end

#seed_import_alias(start_index) ⇒ Object 



382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
 
# File 'lib/milk_tea/core/parser.rb', line 382

def seed_import_alias(start_index)
  cursor = start_index
  last_part = nil

  while cursor < @tokens.length && @tokens[cursor].type != :newline
    token = @tokens[cursor]
    if token.type == :as
      alias_token = @tokens[cursor + 1]
      @known_import_aliases[alias_token.lexeme] = true if type_name_token?(alias_token)
      return cursor
    end

    last_part = token.lexeme if type_name_token?(token)
    cursor += 1
  end

  @known_import_aliases[last_part] = true if last_part
  cursor
end

#seed_known_names ⇒ Object 



333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
 
# File 'lib/milk_tea/core/parser.rb', line 333

def seed_known_names
  MilkTea::BUILTIN_TYPE_NAMES.each { |name| @known_type_names[name] = true }

  depth = 0
  index = 0
  while index < @tokens.length
    token = @tokens[index]
    case token.type
    when :indent
      depth += 1
    when :dedent
      depth -= 1 if depth.positive?
    when :import
      index = seed_import_alias(index + 1) if depth.zero?
    when :function
      if depth.zero?
        name_token = @tokens[index + 1]
        type_param_token = @tokens[index + 2]
        if type_name_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :async
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if type_name_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :foreign
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if type_name_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :struct, :union, :enum, :flags, :opaque, :type, :variant
      if depth.zero?
        name_token = @tokens[index + 1]
        @known_type_names[name_token.lexeme] = true if type_name_token?(name_token)
      end
    end

    index += 1
  end
end

#skip_newlines ⇒ Object 



245
246
247
 
# File 'lib/milk_tea/core/parser.rb', line 245

def skip_newlines
  advance while check(:newline)
end

#type_name_token?(token) ⇒ Boolean

Returns:

  • (Boolean) 


402
403
404
 
# File 'lib/milk_tea/core/parser.rb', line 402

def type_name_token?(token)
  token&.type == :identifier
end

#with_type_param_names(names) ⇒ Object 



305
306
307
308
309
310
311
 
# File 'lib/milk_tea/core/parser.rb', line 305

def with_type_param_names(names)
  saved_names = @current_type_param_names
  @current_type_param_names = @current_type_param_names + names
  yield
ensure
  @current_type_param_names = saved_names
end