Class: Ibex::Codegen::Ruby
Overview
Generates a standalone Ruby parser class from Automaton IR.
rubocop:disable Metrics/ClassLength -- generation stages are split into focused mixins.
Instance Method Summary
collapse
-
#action_references_locations?(production, action) ⇒ Boolean
-
#append_cst_metadata(lines, indent) ⇒ void
-
#append_parameter_initializer(lines) ⇒ void
-
#append_parser_tables(lines, indent, table_set) ⇒ void
-
#append_runtime(lines) ⇒ void
-
#append_tables(lines) ⇒ void
-
#append_user_code(lines, name, indent: 0) ⇒ void
-
#class_parts ⇒ [ Array[String], String ]
-
#compact_production_flags(production) ⇒ Integer
-
#compact_production_metadata_literal ⇒ String
-
#compact_productions_literal ⇒ String
-
#cst? ⇒ Boolean
-
#cst_metadata ⇒ CSTMetadata::metadata
-
#deep_frozen_literal(value) ⇒ String
-
#eager_lexer_feedback_reduction(state) ⇒ IR::reduce_action?
-
#eager_reductions_literal ⇒ String
Reductions that are the sole non-error behavior of a state may run before requesting another lexer token.
-
#external_token_expression(terminal) ⇒ String
-
#foreign_action_sentinel ⇒ String
Kept local so the generic generator does not load the optional Bison importer and its IR validation graph.
-
#generate ⇒ String
-
#grammar_digest_comment ⇒ String
-
#initialize(automaton, table: :compact, embedded: false, line_convert: true, debug: false, line_convert_all: false, omit_action_call: nil, superclass: nil, executable: nil, cst_trivia: :leading, runtime_require: "ibex/runtime", error_messages: {}) ⇒ Ruby
constructor
-
#map_user_code?(name) ⇒ Boolean
-
#packed_integers_literal(values) ⇒ String
-
#packed_signed_integers_literal(values) ⇒ String
-
#plain_productions_literal ⇒ String
-
#production_action_abi_literal(production, generated_action) ⇒ String
-
#production_location_metadata(action) ⇒ [ String, String ]
-
#productions_literal ⇒ String
-
#reject_foreign_actions ⇒ void
-
#shareable_parser_tables? ⇒ Boolean
Custom conversion expressions can return caller-owned mutable or inherently unshareable objects, so only standard token mappings are safe to freeze transitively.
-
#symbol_names_literal ⇒ String
-
#table_literal(table) ⇒ String
-
#token_ids_literal ⇒ String
-
#token_names_literal ⇒ String
-
#uses_locations? ⇒ Boolean
Methods included from RubySyntax
#append_repetition_accessors, #append_syntax, #append_syntax_node, #merge_syntax_fields, #syntax_node_definitions
Methods included from RubyAST
#append_ast, #append_listener, #append_listener_dispatch, #append_visit_children, #append_visitor, #ast_method_name, #ast_node_action_source, #ast_node_definitions
Methods included from RubyLexer
#append_lexer, #append_lexer_actions, #append_lexer_rules, #lexer_action_source, #lexer_rule_literal
#append_value_printer, #append_value_printer_table, #append_value_printers, #value_printer_action_method_source
#append_entry_methods, #append_table_metadata, #entry_table_fields, #recovery_table_fields
#error_message_literal, #error_messages_literal
#action_method?, #action_method_source, #append_action_method, #append_actions, #append_compiled_action_method, #append_composed_action_method, #append_composed_fragment_method, #append_composed_orchestrator, #append_composed_step, #composed_action?, #composed_fragment_name
Constructor Details
#initialize(automaton, table: :compact, embedded: false, line_convert: true, debug: false, line_convert_all: false, omit_action_call: nil, superclass: nil, executable: nil, cst_trivia: :leading, runtime_require: "ibex/runtime", error_messages: {}) ⇒ Ruby
Returns a new instance of Ruby.
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
# File 'lib/ibex/codegen/ruby.rb', line 46
def initialize(automaton, table: :compact, embedded: false, line_convert: true, debug: false,
line_convert_all: false, omit_action_call: nil, superclass: nil, executable: nil,
cst_trivia: :leading, runtime_require: "ibex/runtime", error_messages: {})
@automaton = automaton
@grammar = automaton.grammar
reject_foreign_actions
@table_format = table.to_sym
@embedded = embedded
@line_convert = line_convert
@line_convert_all = line_convert_all
@debug = debug
@omit_action_call = omit_action_call.nil? ? @grammar.options[:omit_action_call] : omit_action_call
@superclass = superclass || @grammar.superclass || "Ibex::Runtime::Parser"
@executable = executable
@cst_trivia = cst_trivia.to_sym
@generated_action_abi = GeneratedActionABI::Cache.new
@runtime_require = runtime_require
@cst_trivia = :leading if @cst_trivia == :attach
unless %i[leading balanced drop].include?(@cst_trivia)
raise ArgumentError, "cst_trivia must be :leading, :balanced, or :drop"
end
@error_messages = error_messages.sort.to_h.freeze
end
|
Instance Method Details
#action_references_locations?(production, action) ⇒ Boolean
426
427
428
429
|
# File 'lib/ibex/codegen/ruby.rb', line 426
def action_references_locations?(production, action)
maximum = action.context_length.positive? ? action.context_length : production.rhs.length
ActionLocations.new(action.code, maximum: maximum, location: action.location).references?
end
|
This method returns an undefined value.
156
157
158
159
160
161
162
163
164
|
# File 'lib/ibex/codegen/ruby.rb', line 156
def append_cst_metadata(lines, indent)
return unless cst?
lines << "#{indent}SYMBOL_NAMES = #{symbol_names_literal}.freeze"
lines << "#{indent}CST_METADATA = #{deep_frozen_literal(cst_metadata)}"
lines << "#{indent}CST_KIND_MODEL = Ibex::Runtime::CST::Kind.new("
lines << "#{indent} CST_METADATA.fetch(:kinds), slots: CST_METADATA.fetch(:slots)"
lines << "#{indent})"
end
|
#append_parameter_initializer(lines) ⇒ void
This method returns an undefined value.
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
|
# File 'lib/ibex/codegen/ruby.rb', line 193
def append_parameter_initializer(lines)
parameters = @grammar.parser_parameters
return if parameters.empty?
keywords = parameters.map { |parameter| "#{parameter[:name]}:" }.join(", ")
lines << " IBEX_PARAMETER_INITIALIZER = Module.new do"
lines << " def initialize(#{keywords}, **_ibex_super_arguments)"
parameters.each { |parameter| lines << " @#{parameter[:name]} = #{parameter[:name]}" }
lines << " super(**_ibex_super_arguments)"
lines << " end"
lines << " end"
lines << " prepend IBEX_PARAMETER_INITIALIZER"
lines << " private_constant :IBEX_PARAMETER_INITIALIZER"
lines << ""
end
|
#append_parser_tables(lines, indent, table_set) ⇒ void
This method returns an undefined value.
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
# File 'lib/ibex/codegen/ruby.rb', line 167
def append_parser_tables(lines, indent, table_set)
lines << "#{indent}PARSER_TABLES = { format_version: PARSER_TABLE_FORMAT_VERSION,"
lines << "#{indent} grammar_digest: GRAMMAR_DIGEST,"
lines << "#{indent} state_count: STATE_COUNT, production_count: PRODUCTION_COUNT,"
lines << "#{indent} uses_locations: #{uses_locations?},"
if @table_format == :compact
default_codes = table_set.default_actions.map { |action| Tables::CompactActions.pack(action) }
lines << "#{indent} compact_fast_driver: true, " \
"compact_action_encoding: :signed, compact_default_actions: #{default_codes.inspect}.freeze,"
end
lines << "#{indent} cst: CST_METADATA, cst_kinds: CST_KIND_MODEL," if cst?
lines << "#{indent} exact_expected_tokens: true," if @grammar.mode == :extended
lines << "#{indent} tokens: TOKEN_IDS, token_names: TOKEN_NAMES, actions: ACTIONS,"
lines << "#{indent} gotos: GOTOS, default_actions: DEFAULT_ACTIONS,"
entries = entry_table_fields
recovery = recovery_table_fields
value_printers = @grammar.value_printers.empty? ? "" : " value_printers: VALUE_PRINTERS,"
lines << "#{indent} productions: PRODUCTIONS,#{entries}#{value_printers} " \
"#{recovery} eager_reductions: EAGER_REDUCTIONS, error_messages: ERROR_MESSAGES }.freeze"
return unless shareable_parser_tables?
lines << "#{indent}Ractor.make_shareable(PARSER_TABLES) " \
"if defined?(Ractor) && Ractor.respond_to?(:make_shareable)"
end
|
#append_runtime(lines) ⇒ void
This method returns an undefined value.
119
120
121
122
123
124
125
126
127
|
# File 'lib/ibex/codegen/ruby.rb', line 119
def append_runtime(lines)
if @embedded
require "ibex/runtime/embedded_source"
lines << Runtime::EmbeddedSource.render
elsif @runtime_require
lines << "require #{@runtime_require.inspect}"
end
lines << ""
end
|
#append_tables(lines) ⇒ void
This method returns an undefined value.
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
# File 'lib/ibex/codegen/ruby.rb', line 130
def append_tables(lines)
table_set = Tables.build(@automaton, format: @table_format)
indent = " "
append_table_metadata(lines, indent)
lines << "#{indent}TOKEN_IDS = #{token_ids_literal}.freeze"
lines << "#{indent}TOKEN_NAMES = #{token_names_literal}.freeze"
append_cst_metadata(lines, indent)
lines << "#{indent}ACTIONS = #{table_literal(table_set.actions)}"
lines << "#{indent}GOTOS = #{table_literal(table_set.gotos)}"
lines << "#{indent}DEFAULT_ACTIONS = #{table_set.default_actions.inspect}.freeze"
eager_type = "Hash[Integer, [:reduce, Integer]]"
lines << "#{indent}eager_reductions = #{eager_reductions_literal} # @type var eager_reductions: #{eager_type}"
lines << "#{indent}EAGER_REDUCTIONS = eager_reductions.freeze #: #{eager_type}"
lines << "#{indent}PRODUCTIONS = #{productions_literal}.freeze"
append_value_printer_table(lines, indent)
declaration = "#{indent}error_messages = #{error_messages_literal}"
type = "Hash[Integer, String | { id: String, message: String }]"
lines << "#{declaration} # @type var error_messages: #{type}"
lines << "#{indent}ERROR_MESSAGES = error_messages.freeze #: #{type}"
append_parser_tables(lines, indent, table_set)
lines << "#{indent}def self.parser_tables = PARSER_TABLES"
lines << "#{indent}DEBUG_ENABLED = #{@debug}"
lines << ""
end
|
#append_user_code(lines, name, indent: 0) ⇒ void
This method returns an undefined value.
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
|
# File 'lib/ibex/codegen/ruby.rb', line 437
def append_user_code(lines, name, indent: 0)
code = @grammar.user_code[name]
return if code.nil? || code.empty?
prefix = " " * indent
chunks = @grammar.user_code_chunks.fetch(name, Array.new(0))
if map_user_code?(name) && !chunks.empty?
chunks.each do |chunk|
location = chunk.location
lines << "#{prefix}eval(#{chunk.code.dump}, binding, #{location[:file].inspect}, #{location[:line]})"
end
elsif @line_convert_all
raise Ibex::Error, "(codegen):1:1: source locations are required to convert #{name} user code"
else
code.lines.each { |line| lines << "#{prefix}#{line.rstrip}" }
end
lines << ""
end
|
#class_parts ⇒ [ Array[String], String ]
462
463
464
465
|
# File 'lib/ibex/codegen/ruby.rb', line 462
def class_parts
parts = @grammar.class_name.split("::")
[parts.take(parts.length - 1), parts.fetch(-1)]
end
|
#compact_production_flags(production) ⇒ Integer
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
|
# File 'lib/ibex/codegen/ruby.rb', line 343
def compact_production_flags(production)
return 0 unless action_method?(production)
value = 0
if @generated_action_abi.positional_values?(production)
value |= Tables::CompactProductions::POSITIONAL_ACTION
elsif @generated_action_abi.values_only?(production)
value |= Tables::CompactProductions::VALUES_ACTION
value |= Tables::CompactProductions::BORROWED_VALUES_ACTION if
@generated_action_abi.borrowed_values?(production)
else
value |= Tables::CompactProductions::LOCATION_ACTION
end
value |= Tables::CompactProductions::COMPOSITION_ACTION if composed_action?(production)
value
end
|
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
|
# File 'lib/ibex/codegen/ruby.rb', line 361
def compact_production_metadata_literal
entries = @grammar.productions.filter_map do |production|
fields = [] fields << "rhs: #{production.rhs.inspect}.freeze" if cst?
action = production.action
if action
fields << "location_context_length: #{action.context_length}" if action.context_length.positive?
names = action.named_refs.to_h { |reference| [reference[:name].to_sym, reference[:index]] }
fields << "location_names: #{names.inspect}.freeze" unless names.empty?
end
next if fields.empty?
"#{production.id} => { #{fields.join(', ')} }.freeze"
end
entries.empty? ? "Hash.new.freeze" : "{ #{entries.join(', ')} }.freeze"
end
|
#compact_productions_literal ⇒ String
333
334
335
336
337
338
339
340
|
# File 'lib/ibex/codegen/ruby.rb', line 333
def compact_productions_literal
lhs_ids = @grammar.productions.map(&:lhs)
lengths = @grammar.productions.map { |production| production.rhs.length }
flags = @grammar.productions.map { |production| compact_production_flags(production) }
metadata = compact_production_metadata_literal
"Ibex::Tables::CompactProductions.packed(#{packed_integers_literal(lhs_ids)}, " \
"#{packed_integers_literal(lengths)}, #{packed_integers_literal(flags)}, metadata: #{metadata})"
end
|
#cst? ⇒ Boolean
432
433
434
|
# File 'lib/ibex/codegen/ruby.rb', line 432
def cst?
@grammar.options[:cst] == true
end
|
218
219
220
|
# File 'lib/ibex/codegen/ruby.rb', line 218
def cst_metadata
@cst_metadata ||= CSTMetadata.new(@grammar, trivia_policy: @cst_trivia).build
end
|
#deep_frozen_literal(value) ⇒ String
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
|
# File 'lib/ibex/codegen/ruby.rb', line 223
def deep_frozen_literal(value)
case value
when Array
"[#{value.map { |item| deep_frozen_literal(item) }.join(', ')}].freeze"
when Hash
entries = value.map do |key, item|
"#{deep_frozen_literal(key)} => #{deep_frozen_literal(item)}"
end
"{ #{entries.join(', ')} }.freeze"
when String
"#{value.inspect}.freeze"
else
value.inspect
end
end
|
#eager_lexer_feedback_reduction(state) ⇒ IR::reduce_action?
395
396
397
398
399
400
401
402
403
404
405
|
# File 'lib/ibex/codegen/ruby.rb', line 395
def eager_lexer_feedback_reduction(state)
candidates = state.actions.values
candidates += [state.default_action] if state.default_action
actions = candidates.reject { |action| action[:type] == :error }.uniq
return unless actions.one?
return unless actions.first[:type] == :reduce
reduction = actions.first production = @grammar.productions.fetch(reduction.fetch(:production))
reduction if production.action&.code&.match?(/\blexer_state\s*=/)
end
|
#eager_reductions_literal ⇒ String
Reductions that are the sole non-error behavior of a state may run
before requesting another lexer token. This enables parser actions to
select lexer_state without speculative lookahead.
382
383
384
385
386
387
388
389
390
391
392
|
# File 'lib/ibex/codegen/ruby.rb', line 382
def eager_reductions_literal
return "{}" unless @grammar.lexer
entries = @automaton.states.filter_map do |state|
action = eager_lexer_feedback_reduction(state)
next unless action
"#{state.id} => [:reduce, #{action.fetch(:production)}].freeze"
end
"{ #{entries.join(', ')} }"
end
|
#external_token_expression(terminal) ⇒ String
274
275
276
277
278
279
280
|
# File 'lib/ibex/codegen/ruby.rb', line 274
def external_token_expression(terminal)
conversion = @grammar.conversions[terminal.name]
return "(#{conversion})" if conversion
return terminal.name if terminal.name.start_with?("'", '"')
":#{terminal.name}"
end
|
#foreign_action_sentinel ⇒ String
Kept local so the generic generator does not load the optional Bison
importer and its IR validation graph.
114
115
116
|
# File 'lib/ibex/codegen/ruby.rb', line 114
def foreign_action_sentinel
"__ibex_foreign_action_c__"
end
|
#generate ⇒ String
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
# File 'lib/ibex/codegen/ruby.rb', line 72
def generate
lines = [] lines << "#!#{@executable}" if @executable
lines.push("# frozen_string_literal: true", "# Generated by Ibex #{}", "")
append_user_code(lines, "header")
append_runtime(lines)
modules, class_name = class_parts
modules.each { |name| lines << "module #{name}" }
lines << "class #{class_name} < #{@superclass}"
append_ast(lines)
append_syntax(lines)
append_tables(lines)
append_parameter_initializer(lines)
append_entry_methods(lines)
append_lexer(lines)
append_value_printers(lines)
append_actions(lines)
append_user_code(lines, "inner", indent: 2)
lines << "end"
modules.reverse_each { lines << "end" }
append_user_code(lines, "footer")
"#{lines.join("\n")}\n"
end
|
468
469
470
|
# File 'lib/ibex/codegen/ruby.rb', line 468
def
@automaton.grammar_digest.delete_prefix("sha256:").chars.first(12).join
end
|
#map_user_code?(name) ⇒ Boolean
457
458
459
|
# File 'lib/ibex/codegen/ruby.rb', line 457
def map_user_code?(name)
@line_convert_all || (@line_convert && name == "inner")
end
|
#packed_integers_literal(values) ⇒ String
254
255
256
|
# File 'lib/ibex/codegen/ruby.rb', line 254
def packed_integers_literal(values)
Tables::PackedIntegers.encode(values).inspect
end
|
#packed_signed_integers_literal(values) ⇒ String
259
260
261
|
# File 'lib/ibex/codegen/ruby.rb', line 259
def packed_signed_integers_literal(values)
Tables::PackedIntegers.encode_signed(values).inspect
end
|
#plain_productions_literal ⇒ String
304
305
306
307
308
309
310
311
312
313
314
315
316
|
# File 'lib/ibex/codegen/ruby.rb', line 304
def plain_productions_literal
entries = @grammar.productions.map do |production|
generated_action = action_method?(production)
action = generated_action ? ":_ibex_action_#{production.id}" : "nil"
action_abi = production_action_abi_literal(production, generated_action)
composition_action = composed_action?(production) ? ", composition_action: true" : ""
cst_rhs = cst? ? ", rhs: #{production.rhs.inspect}.freeze" : ""
location_context, named_locations = production_location_metadata(production.action)
"{ lhs: #{production.lhs}, length: #{production.rhs.length}, action: #{action}" \
"#{action_abi}#{composition_action}#{cst_rhs}#{location_context}#{named_locations} }"
end
"[#{entries.join(', ')}]"
end
|
#production_action_abi_literal(production, generated_action) ⇒ String
319
320
321
322
323
324
325
326
327
328
329
330
|
# File 'lib/ibex/codegen/ruby.rb', line 319
def production_action_abi_literal(production, generated_action)
return "" unless generated_action
return ", location_action: true" unless @generated_action_abi.values_only?(production)
return ", positional_action: true" if @generated_action_abi.positional_values?(production)
borrowed = if @generated_action_abi.borrowed_values?(production)
", borrowed_values_action: true"
else
""
end
", values_action: true#{borrowed}"
end
|
408
409
410
411
412
413
414
415
|
# File 'lib/ibex/codegen/ruby.rb', line 408
def production_location_metadata(action)
return ["", ""] unless action
context = action.context_length.positive? ? ", location_context_length: #{action.context_length}" : ""
names = action.named_refs.to_h { |reference| [reference[:name].to_sym, reference[:index]] }
named = names.empty? ? "" : ", location_names: #{names.inspect}.freeze"
[context, named]
end
|
#productions_literal ⇒ String
297
298
299
300
301
|
# File 'lib/ibex/codegen/ruby.rb', line 297
def productions_literal
return compact_productions_literal if @table_format == :compact
plain_productions_literal
end
|
#reject_foreign_actions ⇒ void
This method returns an undefined value.
99
100
101
102
103
104
105
106
107
108
109
|
# File 'lib/ibex/codegen/ruby.rb', line 99
def reject_foreign_actions
production = @grammar.productions.find do |entry|
entry.action&.code&.include?(foreign_action_sentinel)
end
return unless production
location = production.action&.location
rendered = location ? "#{location[:file]}:#{location[:line]}:#{location[:column]}" : "(codegen):1:1"
raise Ibex::Error,
"#{rendered}: cannot generate Ruby from imported C semantic actions; use analysis commands only"
end
|
#shareable_parser_tables? ⇒ Boolean
Custom conversion expressions can return caller-owned mutable or
inherently unshareable objects, so only standard token mappings are
safe to freeze transitively.
213
214
215
|
# File 'lib/ibex/codegen/ruby.rb', line 213
def shareable_parser_tables?
@grammar.conversions.empty?
end
|
#symbol_names_literal ⇒ String
291
292
293
294
|
# File 'lib/ibex/codegen/ruby.rb', line 291
def symbol_names_literal
entries = @grammar.symbols.map { |symbol| "#{symbol.id} => #{symbol.name.inspect}" }
"{ #{entries.join(', ')} }"
end
|
#table_literal(table) ⇒ String
240
241
242
243
244
245
246
247
248
249
250
251
|
# File 'lib/ibex/codegen/ruby.rb', line 240
def table_literal(table)
if table.is_a?(Tables::CompactActions)
return "Ibex::Tables::CompactActions.packed(#{packed_integers_literal(table.offsets)}, " \
"#{packed_signed_integers_literal(table.codes)}, #{packed_integers_literal(table.checks)}, " \
"row_count: #{table.row_count}, encoding: :signed, column_count: #{table.column_count})"
end
return "#{table.inspect}.freeze" unless table.is_a?(Tables::Compact)
"Ibex::Tables::Compact.packed(#{packed_integers_literal(table.offsets)}, " \
"#{packed_integers_literal(table.values)}, #{packed_integers_literal(table.checks)}, " \
"row_count: #{table.row_count}, dense_width: #{table.dense_width})"
end
|
#token_ids_literal ⇒ String
264
265
266
267
268
269
270
271
|
# File 'lib/ibex/codegen/ruby.rb', line 264
def token_ids_literal
pairs = @grammar.terminals.filter_map do |terminal|
next if terminal.id.zero? || terminal.name == "error"
"#{external_token_expression(terminal)} => #{terminal.id}"
end
"{ #{pairs.join(', ')} }"
end
|
#token_names_literal ⇒ String
283
284
285
286
287
288
|
# File 'lib/ibex/codegen/ruby.rb', line 283
def token_names_literal
entries = @grammar.terminals.map do |terminal|
"#{terminal.id} => #{(terminal.display_name || terminal.name).inspect}"
end
"{ #{entries.join(', ')} }"
end
|
#uses_locations? ⇒ Boolean
418
419
420
421
422
423
|
# File 'lib/ibex/codegen/ruby.rb', line 418
def uses_locations?
cst? || @grammar.productions.any? do |production|
action = production.action
action && (!action.composition.nil? || action_references_locations?(production, action))
end
end
|