Class: MilkTea::Parser

Inherits:
Object
  • Object
show all
Includes:
MilkTea::Parse::Attributes, MilkTea::Parse::Blocks, MilkTea::Parse::Declarations, MilkTea::Parse::Expressions, MilkTea::Parse::Recovery, MilkTea::Parse::Statements, MilkTea::Parse::Types
Defined in:
lib/milk_tea/core/parser.rb

Defined Under Namespace

Classes: ParseRecoveryResult

Constant Summary collapse

BUILTIN_ATTRIBUTE_NAME_LEXEMES = 
%w[packed align].freeze

Constants included from MilkTea::Parse::Recovery

MilkTea::Parse::Recovery::TOP_LEVEL_RECOVERY_START_TYPES

Class Method Summary collapse

Instance Method Summary collapse

Methods included from MilkTea::Parse::Statements

#check_inline_stmt_start?, #check_parallel_block_start?, #check_parallel_for_start?, #check_when_start?, #inline_block_body?, #match_arm_expr_form?, #parse_assignment_or_expression_stmt, #parse_break_stmt, #parse_continue_stmt, #parse_decl_when_arms, #parse_defer_stmt, #parse_destructure_pattern, #parse_else_branch_body, #parse_emit_stmt, #parse_for_stmt, #parse_gather_stmt, #parse_if_branch, #parse_if_stmt, #parse_inline_for_stmt, #parse_inline_if_stmt, #parse_inline_match_stmt, #parse_inline_stmt, #parse_inline_while_stmt, #parse_local_decl, #parse_match_arm, #parse_match_arm_body, #parse_match_arms, #parse_match_stmt, #parse_parallel_block_stmt, #parse_parallel_for_stmt, #parse_pass_stmt, #parse_return_stmt, #parse_statement, #parse_static_assert, #parse_unsafe_stmt, #parse_when_decl, #parse_when_stmt, #parse_while_stmt, #reject_operator_statement_start!

Methods included from MilkTea::Parse::Declarations

#extending_target_type_param_names, #extending_target_type_param_names_from_argument, #parse_attribute_decl, #parse_block_body_safe, #parse_callable_signature, #parse_compiler_flag_directive, #parse_const_decl, #parse_declaration, #parse_enum_decl, #parse_event_decl, #parse_extending_block, #parse_extern_decl, #parse_extern_function_decl, #parse_foreign_decl, #parse_foreign_function_decl, #parse_function_def, #parse_import, #parse_include_directive, #parse_interface_decl, #parse_interface_method_decl, #parse_link_directive, #parse_method_def, #parse_method_kind, #parse_method_like_decl_head, #parse_opaque_decl, #parse_optional_explicit_c_name, #parse_raw_module_body, #parse_raw_module_declaration, #parse_raw_module_directive, #parse_struct_decl, #parse_struct_decl_params, #parse_struct_member, #parse_type_alias_decl, #parse_union_decl, #parse_var_decl, #parse_variant_decl, #parse_visibility, #raw_module_declaration_error_message, #raw_module_directive_start?

Methods included from MilkTea::Parse::Expressions

#adjacent_tokens?, #aggregate_specialization_target?, #builtin_specialization_target?, #definite_type_argument?, #explicit_specialization_argument?, #generic_callable_specialization_target?, #imported_member_specialization_target?, #parse_additive, #parse_adjacent_string_literal, #parse_alignof_expr, #parse_and, #parse_bitwise_and, #parse_bitwise_or, #parse_bitwise_xor, #parse_call_argument, #parse_call_arguments, #parse_comparison, #parse_diagnostic_hint?, #parse_embedded_expression, #parse_equality, #parse_expression, #parse_format_spec, #parse_format_string_literal, #parse_format_string_part, #parse_if_expression, #parse_is, #parse_left_associative, #parse_match_arm_pattern, #parse_match_expression, #parse_match_expression_arm, #parse_match_expression_arms, #parse_multiplicative, #parse_not, #parse_offsetof_expr, #parse_or, #parse_postfix, #parse_primary, #parse_proc_expr, #parse_range, #parse_shift, #parse_sizeof_expr, #parse_unary, #parse_unsafe_expression, #postfix_bracket_starts_specialization?, #potential_named_literal_type_argument?, #specialization_call_target?, #specialization_target?, #specialization_value_target?, #try_parse_prefix_cast_expression, #try_parse_specialization

Methods included from MilkTea::Parse::Attributes

#consume_attribute_name_component, #parse_attribute_application, #parse_attribute_applications, #parse_attribute_name, #parse_struct_layout_attributes, #reject_attributes!

Methods included from MilkTea::Parse::Types

#parse_callable_type_ref, #parse_declaration_type_params, #parse_dyn_type_ref, #parse_foreign_param, #parse_foreign_params, #parse_function_type_param, #parse_function_type_ref, #parse_implements_clause, #parse_param, #parse_param_default_value, #parse_parameter_list, #parse_params, #parse_proc_type_ref, #parse_qualified_name_with_type_arguments, #parse_signature_params, #parse_tuple_type_ref, #parse_type_argument, #parse_type_param_constraint, #parse_type_param_constraints, #parse_type_ref

Methods included from MilkTea::Parse::Recovery

#recover_match_arm_block, #recover_statement_block_body, #recovery_error_block_stmt, #recovery_error_expr, #recovery_error_stmt, #recovery_statement_header_type, #synchronize_to_match_arm_boundary, #synchronize_to_statement_boundary, #synchronize_to_top_level_boundary, #top_level_recovery_start?

Methods included from MilkTea::Parse::Blocks

#parse_and_dedent_block_body, #parse_block, #parse_block_body, #parse_declaration_block, #parse_named_block, #parse_statement_block_body, #unexpected_statement_block_indent_token

Constructor Details

#initialize(tokens, path: nil) ⇒ Parser

Returns a new instance of Parser.



65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/milk_tea/core/parser.rb', line 65

def initialize(tokens, path: nil)
  @tokens = tokens.is_a?(SyntaxTokenStream) ? tokens : SyntaxTokenStream.new(tokens)
  @path = path
  @current = 0
  @known_type_names = {}
  @known_import_aliases = {}
  @known_generic_callable_names = {}
  @current_type_param_names = []
  @in_inline_block_body = false
  seed_known_names
end

Class Method Details

.parse(source = nil, path: nil, tokens: nil) ⇒ Object 



49
50
51
52
 
# File 'lib/milk_tea/core/parser.rb', line 49

def self.parse(source = nil, path: nil, tokens: nil)
  token_stream = tokens || Lexer.lex(source, path: path)
  new(token_stream, path: path).parse
end

.parse_collecting_errors(source = nil, path: nil, tokens: nil) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
 
# File 'lib/milk_tea/core/parser.rb', line 54

def self.parse_collecting_errors(source = nil, path: nil, tokens: nil)
  if tokens
    return new(tokens, path: path).parse_collecting_errors
  end

  lex_errors = []
  token_stream = Lexer.lex(source, path: path, recovery_errors: lex_errors)
  parse_result = new(token_stream, path: path).parse_collecting_errors
  ParseRecoveryResult.new(ast: parse_result.ast, errors: lex_errors + parse_result.errors)
end

Instance Method Details

#advance ⇒ Object 



222
223
224
225
 
# File 'lib/milk_tea/core/parser.rb', line 222

def advance
  @current += 1 unless eof?
  previous
end

#block_expression?(expression) ⇒ Boolean

Returns:

  • (Boolean) 


253
254
255
 
# File 'lib/milk_tea/core/parser.rb', line 253

def block_expression?(expression)
  expression.is_a?(AST::ProcExpr) || expression.is_a?(AST::MatchExpr) || expression.is_a?(AST::IfExpr)
end

#builtin_attribute_identifier?(token) ⇒ Boolean

Returns:

  • (Boolean) 


291
292
293
 
# File 'lib/milk_tea/core/parser.rb', line 291

def builtin_attribute_identifier?(token)
  token&.type == :identifier && BUILTIN_ATTRIBUTE_NAME_LEXEMES.include?(token.lexeme)
end

#check(type) ⇒ Object 



199
200
201
202
203
 
# File 'lib/milk_tea/core/parser.rb', line 199

def check(type)
  return false if eof?

  peek.type == type
end

#check_name ⇒ Object 



205
206
207
 
# File 'lib/milk_tea/core/parser.rb', line 205

def check_name
  !eof? && peek.type == :identifier
end

#check_next(type) ⇒ Object 



209
210
211
212
213
 
# File 'lib/milk_tea/core/parser.rb', line 209

def check_next(type)
  return false if (@current + 1) >= @tokens.length

  @tokens[@current + 1].type == type
end

#consume(type, message) ⇒ Object 



165
166
167
168
169
 
# File 'lib/milk_tea/core/parser.rb', line 165

def consume(type, message)
  return advance if check(type)

  raise error(%i[rparen rbracket rbrace].include?(type) ? previous : peek, message)
end

#consume_end_of_statement ⇒ Object 



247
248
249
250
251
 
# File 'lib/milk_tea/core/parser.rb', line 247

def consume_end_of_statement
  return if @in_inline_block_body

  consume(:newline, "expected end of statement")
end

#consume_name(message) ⇒ Object 



171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/milk_tea/core/parser.rb', line 171

def consume_name(message)
  if keyword_token?(peek)
    name_role = message.sub(/\A(?:expected|required)\s+/, "")
    clearer_message = if name_role == message
                        message
                      else
                        "keyword '#{peek.lexeme}' cannot be used as #{name_role}"
                      end
    raise error(peek, clearer_message)
  end

  consume(:identifier, message)
end

#consume_name_allowing_keywords(message) ⇒ Object 



185
186
187
188
189
190
191
192
193
 
# File 'lib/milk_tea/core/parser.rb', line 185

def consume_name_allowing_keywords(message)
  if check(:identifier)
    advance
  elsif keyword_token?(peek)
    advance
  else
    raise error(peek, message)
  end
end

#consume_path_component(message) ⇒ Object 



265
266
267
268
269
 
# File 'lib/milk_tea/core/parser.rb', line 265

def consume_path_component(message)
  return advance if !eof? && (peek.type == :identifier || Token::KEYWORDS.value?(peek.type))

  raise error(peek, message)
end

#eof? ⇒ Boolean

Returns:

  • (Boolean) 


227
228
229
 
# File 'lib/milk_tea/core/parser.rb', line 227

def eof?
  peek.type == :eof
end

#error(token, message) ⇒ Object 



239
240
241
 
# File 'lib/milk_tea/core/parser.rb', line 239

def error(token, message)
  ParseError.new(message, token:, path: @path)
end

#external_file_header? ⇒ Boolean

Returns:

  • (Boolean) 


151
152
153
154
155
156
 
# File 'lib/milk_tea/core/parser.rb', line 151

def external_file_header?
  return false unless check(:external)

  next_token = @tokens[@current + 1]
  next_token.nil? || %i[newline eof].include?(next_token.type)
end

#foreign_param_qualifier_mode? ⇒ Boolean

Returns:

  • (Boolean) 


285
286
287
288
289
 
# File 'lib/milk_tea/core/parser.rb', line 285

def foreign_param_qualifier_mode?
  return false unless %i[out in inout consuming].include?(peek.type)

  @tokens[@current + 1]&.type == :identifier
end

#keyword_token?(token) ⇒ Boolean

Returns:

  • (Boolean) 


195
196
197
 
# File 'lib/milk_tea/core/parser.rb', line 195

def keyword_token?(token)
  token && Token::KEYWORDS.key?(token.lexeme)
end

#known_type_like_name?(name) ⇒ Boolean

Returns:

  • (Boolean) 


295
296
297
 
# File 'lib/milk_tea/core/parser.rb', line 295

def known_type_like_name?(name)
  @known_type_names.key?(name) || @known_import_aliases.key?(name) || @current_type_param_names.include?(name)
end

#match(*types) ⇒ Object 



158
159
160
161
162
163
 
# File 'lib/milk_tea/core/parser.rb', line 158

def match(*types)
  return false unless types.any? { |type| check(type) }

  advance
  true
end

#match_name ⇒ Object 



215
216
217
218
219
220
 
# File 'lib/milk_tea/core/parser.rb', line 215

def match_name
  return false unless check_name

  advance
  true
end

#matching_rbracket_index(start_index) ⇒ Object 



380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
 
# File 'lib/milk_tea/core/parser.rb', line 380

def matching_rbracket_index(start_index)
  depth = 0
  index = start_index

  while index < @tokens.length
    case @tokens[index].type
    when :lbracket
      depth += 1
    when :rbracket
      depth -= 1
      return index if depth.zero?
    end
    index += 1
  end

  nil
end

#parse ⇒ Object 



77
78
79
 
# File 'lib/milk_tea/core/parser.rb', line 77

def parse
  parse_source_file
end

#parse_collecting_errors ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/milk_tea/core/parser.rb', line 81

def parse_collecting_errors
  errors = @recovery_errors ? @recovery_errors.dup : []
  previous_recovery_errors = @recovery_errors
  @recovery_errors = errors
  ast = parse_source_file(errors:)
  ParseRecoveryResult.new(ast:, errors:)
rescue ParseError => e
  errors ||= []
  errors << e
  ParseRecoveryResult.new(ast: nil, errors:)
ensure
  @recovery_errors = previous_recovery_errors
end

#parse_comma_separated_until(closing_type) ⇒ Object 



271
272
273
274
275
276
277
278
279
280
281
282
283
 
# File 'lib/milk_tea/core/parser.rb', line 271

def parse_comma_separated_until(closing_type)
  items = []

  unless check(closing_type)
    loop do
      items << yield
      break unless match(:comma)
      break if check(closing_type)
    end
  end

  items
end

#parse_qualified_name ⇒ Object 



257
258
259
260
261
262
263
 
# File 'lib/milk_tea/core/parser.rb', line 257

def parse_qualified_name
  parts = [consume_path_component("expected identifier").lexeme]
  while match(:dot)
    parts << consume_path_component("expected identifier after '.'").lexeme
  end
  AST::QualifiedName.new(parts:)
end

#parse_source_file(errors: nil) ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/milk_tea/core/parser.rb', line 97

def parse_source_file(errors: nil)
  skip_newlines

  module_name = nil
  module_kind = :module
  module_line = nil
  imports = []
  directives = []
  declarations = []

  if external_file_header?
    advance
    module_line = previous.line
    module_kind = :raw_module
    consume(:newline, "expected newline after external") unless eof?
    skip_newlines
    imports, directives, declarations = parse_raw_module_body(errors:)
    skip_newlines
    raise error(peek, "expected end of file after external declarations") unless eof?

    return AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
  end

  while match(:import)
    if errors
      begin
        imports << parse_import
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      imports << parse_import
    end
    skip_newlines
  end

  until eof?
    if errors
      begin
        declarations << parse_declaration
      rescue ParseError => e
        errors << e
        synchronize_to_top_level_boundary
      end
    else
      declarations << parse_declaration
    end
    skip_newlines
  end

  AST.assign_node_ids(AST::SourceFile.new(module_name:, module_kind:, imports:, directives:, declarations:, line: module_line))
end

#peek ⇒ Object 



231
232
233
 
# File 'lib/milk_tea/core/parser.rb', line 231

def peek
  @tokens[@current]
end

#previous ⇒ Object 



235
236
237
 
# File 'lib/milk_tea/core/parser.rb', line 235

def previous
  @tokens[@current - 1]
end

#seed_import_alias(start_index) ⇒ Object 



356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
 
# File 'lib/milk_tea/core/parser.rb', line 356

def seed_import_alias(start_index)
  cursor = start_index
  last_part = nil

  while cursor < @tokens.length && @tokens[cursor].type != :newline
    token = @tokens[cursor]
    if token.type == :as
      alias_token = @tokens[cursor + 1]
      @known_import_aliases[alias_token.lexeme] = true if type_name_token?(alias_token)
      return cursor
    end

    last_part = token.lexeme if type_name_token?(token)
    cursor += 1
  end

  @known_import_aliases[last_part] = true if last_part
  cursor
end

#seed_known_names ⇒ Object 



307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
 
# File 'lib/milk_tea/core/parser.rb', line 307

def seed_known_names
  MilkTea::BUILTIN_TYPE_NAMES.each { |name| @known_type_names[name] = true }

  depth = 0
  index = 0
  while index < @tokens.length
    token = @tokens[index]
    case token.type
    when :indent
      depth += 1
    when :dedent
      depth -= 1 if depth.positive?
    when :import
      index = seed_import_alias(index + 1) if depth.zero?
    when :function
      if depth.zero?
        name_token = @tokens[index + 1]
        type_param_token = @tokens[index + 2]
        if type_name_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :async
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if type_name_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :foreign
      if depth.zero? && @tokens[index + 1]&.type == :function
        name_token = @tokens[index + 2]
        type_param_token = @tokens[index + 3]
        if type_name_token?(name_token) && type_param_token&.type == :lbracket
          @known_generic_callable_names[name_token.lexeme] = true
        end
      end
    when :struct, :union, :enum, :flags, :opaque, :type, :variant
      if depth.zero?
        name_token = @tokens[index + 1]
        @known_type_names[name_token.lexeme] = true if type_name_token?(name_token)
      end
    end

    index += 1
  end
end

#skip_newlines ⇒ Object 



243
244
245
 
# File 'lib/milk_tea/core/parser.rb', line 243

def skip_newlines
  advance while check(:newline)
end

#type_name_token?(token) ⇒ Boolean

Returns:

  • (Boolean) 


376
377
378
 
# File 'lib/milk_tea/core/parser.rb', line 376

def type_name_token?(token)
  token&.type == :identifier
end

#with_type_param_names(names) ⇒ Object 



299
300
301
302
303
304
305
 
# File 'lib/milk_tea/core/parser.rb', line 299

def with_type_param_names(names)
  saved_names = @current_type_param_names
  @current_type_param_names = @current_type_param_names + names
  yield
ensure
  @current_type_param_names = saved_names
end