Module: Ibex::Runtime::GeneratedLexer

Defined in:
lib/ibex/runtime/generated_lexer.rb,
sig/ibex/runtime/generated_lexer.rbs

Overview

Runtime contract mixed into parser classes that declare a lexer. rubocop:disable Metrics/ModuleLength -- matching, actions, state, and positions share one session invariant.

Constant Summary collapse

NO_EMISSION =

Signature:

  • Object

Returns:

  • (Object) 
Object.new.freeze
EMPTY_GREEN_TRIVIA =

Returns:

empty_green_trivia.freeze

Instance Method Summary collapse

Instance Method Details

#advance_lexer_position(lexeme) ⇒ void

This method returns an undefined value.

RBS:

  • (String lexeme) -> void

Parameters:

  • lexeme (String) 


504
505
506
507
508
509
510
511
512
513
514
515
516
517
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 504

def advance_lexer_position(lexeme)
  lexeme.scan(/\X/).each do |value|
    grapheme = value.is_a?(String) ? value : value.join
    @lexer_byte_offset += grapheme.bytesize
    if grapheme.include?("\n")
      @lexer_line += 1
      @lexer_column = 1
      @lexer_byte_column = 1
    else
      @lexer_column += 1
      @lexer_byte_column += grapheme.bytesize
    end
  end
end

#apply_lexer_rule(rule, lexeme, location) ⇒ [ Object?, Object?, Hash[Symbol, Object?] ]?

RBS:

  • (Hash[Symbol, untyped] rule, String lexeme, Hash[Symbol, Object?] location) -> [Object?, Object?, Hash[Symbol, Object?]]?

Parameters:

  • rule (Hash[Symbol, untyped])
  • lexeme (String)
  • location (Hash[Symbol, Object?])

Returns:

  • ([ Object?, Object?, Hash[Symbol, Object?] ], nil) 


287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 287

def apply_lexer_rule(rule, lexeme, location)
  @lexer_lexeme = lexeme
  @lexer_emission = NO_EMISSION
  @lexer_skip_requested = rule.fetch(:kind) == :skip
  action = rule[:action]
  value = action ? __send__(action, lexeme) : lexeme
  emission = @lexer_emission
  if emission.is_a?(Array)
    location[:ibex_cst_text] = lexeme
    attached = attach_cst_trivia(location)
    @lexer_has_token = true
    return [emission.fetch(0), emission.fetch(1), attached.freeze]
  end

  if @lexer_skip_requested
    retain_cst_trivia(lexeme, location.freeze)
    return nil
  end
  if rule.fetch(:kind) == :token
    location[:ibex_cst_text] = lexeme
    attached = attach_cst_trivia(location)
    @lexer_has_token = true
    return [rule.fetch(:token), value, attached.freeze]
  end

  location.freeze
  raise ParseError, "#{location.fetch(:file)}:#{location.fetch(:line)}:" \
                    "#{location.fetch(:column)}: on lexer rule did not emit or skip"
ensure
  @lexer_lexeme = nil
end

#attach_cst_trivia(location) ⇒ Hash[Symbol, Object?]

RBS:

  • (Hash[Symbol, Object?] location) -> Hash[Symbol, Object?]

Parameters:

  • location (Hash[Symbol, Object?])

Returns:

  • (Hash[Symbol, Object?]) 


355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 355

def attach_cst_trivia(location)
  policy = cst_trivia_policy
  trivia = @lexer_pending_green_trivia
  return location if trivia.empty? || policy == :drop

  @lexer_pending_green_trivia = []
  if policy == :balanced && @lexer_has_token
    trailing, leading = split_balanced_trivia(trivia)
  else
    leading = trivia.freeze
    trailing = EMPTY_GREEN_TRIVIA
  end
  location[:leading_trivia] = leading.freeze
  location[:cst_previous_trailing] = trailing.freeze
  location
end

#configure_lexer_cst ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



396
397
398
399
400
401
402
403
404
405
406
407
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 396

def configure_lexer_cst
  tables = parser_tables
  config = tables[:cst]
  if config.is_a?(Hash)
    @lexer_cst_trivia_policy = config.fetch(:trivia_policy)
    @lexer_cst_trivia_kinds = config.fetch(:kinds).fetch(:trivia)
    return
  end

  @lexer_cst_trivia_policy = :drop
  @lexer_cst_trivia_kinds = nil
end

#consume_lexer_match(input, lexeme) ⇒ Hash[Symbol, untyped]

RBS:

  • (LexerInput input, String lexeme) -> Hash[Symbol, untyped]

Parameters:

Returns:

  • (Hash[Symbol, untyped]) 


270
271
272
273
274
275
276
277
278
279
280
281
282
283
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 270

def consume_lexer_match(input, lexeme)
  start = lexer_position
  input.consume(lexeme)
  advance_lexer_position(lexeme)
  finish = lexer_position
  {
    file: @lexer_file, line: start.fetch(:line), column: start.fetch(:column),
    grapheme_column: start.fetch(:column), byte_column: start.fetch(:byte_column),
    end_line: finish.fetch(:line), end_column: finish.fetch(:column),
    end_grapheme_column: finish.fetch(:column), end_byte_column: finish.fetch(:byte_column),
    start_byte: start.fetch(:byte), end_byte: finish.fetch(:byte),
    source_line: input.source_line(start.fetch(:line))
  }
end

#cst_source_text ⇒ CST::SourceText

RBS:

  • () -> CST::SourceText

Returns:



438
439
440
441
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 438

def cst_source_text
  input = @lexer_input || raise(ParseError, "(lexer):1:1: lexer input is unavailable")
  CST::SourceText.new(input.source_bytes.byteslice(0, @lexer_byte_offset || 0) || "".b, file: @lexer_file)
end

#cst_trivia_kind(text) ⇒ Integer

RBS:

  • (String text) -> Integer

Parameters:

  • text (String)

Returns:

  • (Integer) 


379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 379

def cst_trivia_kind(text)
  kinds = @lexer_cst_trivia_kinds || raise(ParseError, "(lexer):1:1: CST trivia kinds are unavailable")
  name = if text.include?("\n")
           "newline"
         elsif text.match?(/\A[[:space:]]+\z/)
           "whitespace"
         elsif text.start_with?("//", "#")
           "line_comment"
         elsif text.start_with?("/*")
           "block_comment"
         else
           "custom_skip"
         end
  kinds.fetch(name)
end

#cst_trivia_policy ⇒ Symbol

RBS:

  • () -> Symbol

Returns:

  • (Symbol) 


373
374
375
376
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 373

def cst_trivia_policy
  configure_lexer_cst unless defined?(@lexer_cst_trivia_policy)
  @lexer_cst_trivia_policy
end

#emit(token, value = NO_EMISSION) ⇒ Object?

Emit a token from a lexer action.

RBS:

  • (Object? token, ?Object? value) -> Object?

Parameters:

  • token (Object, nil)
  • value (Object, nil) (defaults to: NO_EMISSION)

Returns:

  • (Object, nil) 


321
322
323
324
325
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 321

def emit(token, value = NO_EMISSION)
  actual = value.equal?(NO_EMISSION) ? @lexer_lexeme : value
  @lexer_emission = [token, actual]
  actual
end

#ensure_lexer_data?(input) ⇒ Boolean

RBS:

  • (LexerInput input) -> bool

Parameters:

Returns:

  • (Boolean) 


221
222
223
224
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 221

def ensure_lexer_data?(input)
  input.read_more? while input.buffer.empty? && !input.eof?
  !input.buffer.empty?
end

#green_trivia_values(values) ⇒ Array[CST::GreenTrivia]

RBS:

  • (Array[CST::GreenTrivia] values) -> Array[CST::GreenTrivia]

Parameters:

Returns:



451
452
453
454
455
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 451

def green_trivia_values(values)
  result = [] #: Array[CST::GreenTrivia]
  values.each { |item| result << item if item.is_a?(CST::GreenTrivia) }
  result
end

#lex(source, file: "(input)") ⇒ self

Reset the generated lexer to the beginning of an input.

RBS:

  • (String | IO | Fiber source, ?file: String) -> self

Parameters:

  • source (String, IO, Fiber)
  • file: (String) (defaults to: "(input)")

Returns:

  • (self) 


33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 33

def lex(source, file: "(input)")
  @lexer_input = LexerInput.new(source)
  @lexer_file = file
  @lexer_states = ["INITIAL"]
  @lexer_line = 1
  @lexer_column = 1
  @lexer_byte_column = 1
  @lexer_byte_offset = 0
  @lexer_lexeme = nil
  @lexer_emission = NO_EMISSION
  @lexer_skip_requested = false
  @lexer_pending_green_trivia = []
  configure_lexer_cst
  @lexer_has_token = false
  self
end

#lexeme ⇒ String?

Return the text consumed by the active lexer action.

RBS:

  • () -> String?

Returns:

  • (String, nil) 


336
337
338
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 336

def lexeme
  @lexer_lexeme
end

#lexer_position ⇒ Hash[Symbol, Integer]

RBS:

  • () -> Hash[Symbol, Integer]

Returns:

  • (Hash[Symbol, Integer]) 


484
485
486
487
488
489
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 484

def lexer_position
  {
    line: @lexer_line || 1, column: @lexer_column || 1,
    byte_column: @lexer_byte_column || 1, byte: @lexer_byte_offset || 0
  }
end

#lexer_rules ⇒ Array[Hash[Symbol, untyped]]

RBS:

  • () -> Array[Hash[Symbol, untyped]]

Returns:

  • (Array[Hash[Symbol, untyped]]) 


249
250
251
252
253
254
255
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 249

def lexer_rules
  tables = self.class.const_get(:LEXER_RULES_BY_STATE)
  rules = tables[lexer_state]
  return rules if rules

  raise ParseError, "(lexer):1:1: missing rules for lexer state #{lexer_state}"
end

#lexer_state ⇒ Symbol

Return the current named lexer state.

RBS:

  • () -> Symbol

Returns:

  • (Symbol) 


94
95
96
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 94

def lexer_state
  (@lexer_states || ["INITIAL"]).last.to_sym
end

#lexer_state=(state) ⇒ Symbol

Replace the current named lexer state.

RBS:

  • (Symbol | String state) -> Symbol

Parameters:

  • state (Symbol, String)

Returns:

  • (Symbol) 


100
101
102
103
104
105
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 100

def lexer_state=(state)
  name = validate_lexer_state(state)
  states = @lexer_states ||= ["INITIAL"]
  states[-1] = name
  name.to_sym
end

#lexer_zero_width_location ⇒ Hash[Symbol, Object?]

RBS:

  • () -> Hash[Symbol, Object?]

Returns:

  • (Hash[Symbol, Object?]) 


492
493
494
495
496
497
498
499
500
501
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 492

def lexer_zero_width_location
  position = lexer_position
  {
    file: @lexer_file || "(input)", line: position.fetch(:line), column: position.fetch(:column),
    grapheme_column: position.fetch(:column), byte_column: position.fetch(:byte_column),
    end_line: position.fetch(:line), end_column: position.fetch(:column),
    end_grapheme_column: position.fetch(:column), end_byte_column: position.fetch(:byte_column),
    start_byte: position.fetch(:byte), end_byte: position.fetch(:byte)
  }
end

#next_token ⇒ [ Object?, Object?, Hash[Symbol, Object?] ]

Pull one token using longest match and declaration-order tie breaking.

RBS:

  • () -> [Object?, Object?, Hash[Symbol, Object?]]

Returns:

  • ([ Object?, Object?, Hash[Symbol, Object?] ]) 


109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 109

def next_token
  input = @lexer_input
  raise ParseError, "(lexer):1:1: call parse or lex before next_token" unless input

  loop do
    unless ensure_lexer_data?(input)
      location = lexer_zero_width_location
      location[:ibex_lexer_start_state] = lexer_state
      location = attach_cst_trivia(location) if cst_enabled?
      return [nil, nil, location.freeze]
    end

    rule, lexeme = select_lexer_match(input)
    raise_lexer_no_match(input) unless rule && lexeme

    location = consume_lexer_match(input, lexeme)
    location[:ibex_lexer_start_state] = lexer_state
    emitted = apply_lexer_rule(rule, lexeme, location)
    return emitted if emitted
  end
end

#parse(source, file: "(input)") ⇒ Object?

Lex and semantically parse one String, IO, or Fiber source. Parser and generated lexer actions execute. Loading the generated file may also execute user sections; this trusted application path is not a sandbox.

RBS:

  • (String | IO | Fiber source, ?file: String) -> Object?

Parameters:

  • source (String, IO, Fiber)
  • file: (String) (defaults to: "(input)")

Returns:

  • (Object, nil) 


55
56
57
58
59
60
61
62
63
64
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 55

def parse(source, file: "(input)")
  lex(source, file: file)
  parser = self #: Parser
  parser.do_parse
rescue ParseError => e
  raise unless parser_tables[:cst].is_a?(Hash)

  parser = self #: Parser
  parser.__send__(:cst_lexical_failure, e)
end

#parse_syntax(source, file: "(input)") ⇒ CST::SyntaxResult

Parse without executing parser production actions. Generated lexer actions still run because they define tokenization and lexer state. Loading user sections and running lexer actions are trusted application boundaries, so this method is not a no-user-code sandbox.

RBS:

  • (String source, ?file: String) -> CST::SyntaxResult

Parameters:

  • source (String)
  • file: (String) (defaults to: "(input)")

Returns:



88
89
90
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 88

def parse_syntax(source, file: "(input)")
  parse_syntax_with_cache(CST::SourceText.new(source, file: file), CST::NodeCache.new)
end

#parse_syntax_with_cache(source_text, cache) ⇒ CST::SyntaxResult

RBS:

  • (CST::SourceText source_text, CST::NodeCache cache) -> CST::SyntaxResult

Parameters:

Returns:



134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 134

def parse_syntax_with_cache(source_text, cache)
  parser = self #: Parser
  parser.__send__(:with_syntax_only, cache) do
    lex(source_text.text, file: source_text.file || "(input)")
    value = begin
      parser.do_parse
    rescue ParseError => e
      raise unless parser_tables[:cst].is_a?(Hash)

      parser.__send__(:cst_lexical_failure, e)
    end
    parsed = parser.__send__(:syntax_parse_result, value)
    CST::SyntaxResult.new(
      syntax_root: parsed.syntax_root,
      diagnostics: parsed.diagnostics,
      reused_ratio: 0.0
    )
  end
end

#parse_with_syntax(source, file: "(input)") ⇒ CST::ParseResult

Parse one generated-lexer source and return its semantic and syntax results. Parser and generated lexer actions execute. This trusted application path is not a sandbox.

RBS:

  • (String | IO | Fiber source, ?file: String) -> CST::ParseResult

Parameters:

  • source (String, IO, Fiber)
  • file: (String) (defaults to: "(input)")

Returns:



70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 70

def parse_with_syntax(source, file: "(input)")
  lex(source, file: file)
  parser = self #: Parser
  value = parser.do_parse
  parser.__send__(:syntax_parse_result, value)
rescue ParseError => e
  raise unless parser_tables[:cst].is_a?(Hash)

  parser = self #: Parser
  value = parser.__send__(:cst_lexical_failure, e)
  parser.__send__(:syntax_parse_result, value)
end

#pop_state ⇒ Symbol

Pop the current named lexer state.

RBS:

  • () -> Symbol

Returns:

  • (Symbol) 


467
468
469
470
471
472
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 467

def pop_state
  states = @lexer_states ||= ["INITIAL"]
  raise ParseError, "(lexer):1:1: cannot pop the INITIAL lexer state" if states.one?

  (states.pop || "INITIAL").to_sym
end

#push_state(state) ⇒ Symbol

Push a named lexer state.

RBS:

  • (Symbol | String state) -> Symbol

Parameters:

  • state (Symbol, String)

Returns:

  • (Symbol) 


459
460
461
462
463
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 459

def push_state(state)
  name = validate_lexer_state(state)
  (@lexer_states ||= ["INITIAL"]) << name
  name.to_sym
end

#raise_lexer_no_match(input) ⇒ bot

RBS:

  • (LexerInput input) -> bot

Parameters:

Returns:

  • (bot) 


258
259
260
261
262
263
264
265
266
267
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 258

def raise_lexer_no_match(input)
  location = lexer_zero_width_location
  location[:ibex_cst_unmatched_text] = input.buffer.dup.freeze
  location.freeze
  excerpt = input.buffer.slice(0, 16)
  raise ParseError.new(
    token_name: "lexer input", token_value: excerpt, location: location,
    detail: "no lexer rule matches #{excerpt.inspect} in state #{lexer_state}"
  )
end

#replace_previous_scanned_trailing(tokens, location, cache) ⇒ void

This method returns an undefined value.

RBS:

  • (Array[CST::GreenToken] tokens, Object? location, CST::NodeCache cache) -> void

Parameters:



197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 197

def replace_previous_scanned_trailing(tokens, location, cache)
  trailing = scanned_green_trivia(location, :cst_previous_trailing)
  previous = tokens.last
  return if trailing.empty? || !previous

  tokens[-1] = cache.intern_token_fields(
    kind: previous.kind,
    text: previous.text,
    leading: previous.leading,
    trailing: previous.trailing + trailing,
    flags: previous.flags,
    expected_kind: previous.expected_kind
  )
end

#retain_cst_trivia(text, _location) ⇒ void

This method returns an undefined value.

RBS:

  • (String text, Hash[Symbol, Object?] location) -> void

Parameters:

  • text (String)
  • location (Hash[Symbol, Object?]) 


341
342
343
344
345
346
347
348
349
350
351
352
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 341

def retain_cst_trivia(text, _location)
  return if cst_trivia_policy == :drop

  kind = cst_trivia_kind(text)
  cache = @green_cache
  value = if cache
            cache.intern_trivia_fields(kind: kind, text: text)
          else
            CST::GreenTrivia.new(kind: kind, text: text)
          end
  @lexer_pending_green_trivia << value
end

#scan_syntax_with_cache(source_text, cache) ⇒ CST::LexedSyntax

Run the generated lexer once and materialize its exact Green token stream.

RBS:

  • (CST::SourceText source_text, CST::NodeCache cache) -> CST::LexedSyntax

Parameters:

Returns:



156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 156

def scan_syntax_with_cache(source_text, cache)
  lex(source_text.text, file: source_text.file || "(input)")
  raw_tokens = [] #: Array[Array[Object?]]
  green_tokens = [] #: Array[CST::GreenToken]
  states = [] #: Array[Symbol]
  loop do
    external, value, location = next_token
    replace_previous_scanned_trailing(green_tokens, location, cache)
    token_id = external.nil? || external == false ? Parser::EOF_TOKEN : __send__(:internal_token_id, external)
    green = scanned_green_token(token_id, value, location, cache)
    raw_tokens << [external, value, location]
    green_tokens << green
    state = location[:ibex_lexer_start_state] if location.is_a?(Hash)
    states << (state ? state.to_s.to_sym : :INITIAL)
    break if token_id == Parser::EOF_TOKEN
  end
  offsets = [] #: Array[Integer]
  offset = 0
  green_tokens.each do |token|
    offsets << offset
    offset += token.full_width
  end
  memo = CST::TokenMemo.new(tokens: green_tokens, offsets: offsets, states: states)
  CST::LexedSyntax.new(raw_tokens: raw_tokens, memo: memo)
end

#scanned_green_token(token_id, value, location, cache) ⇒ CST::GreenToken

RBS:

  • (Integer token_id, Object? value, Object? location, CST::NodeCache cache) -> CST::GreenToken

Parameters:

  • token_id (Integer)
  • value (Object, nil)
  • location (Object, nil)
  • cache (CST::NodeCache)

Returns:



183
184
185
186
187
188
189
190
191
192
193
194
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 183

def scanned_green_token(token_id, value, location, cache)
  leading = scanned_green_trivia(location, :leading_trivia)
  text = if location.is_a?(Hash) && location[:ibex_cst_text].is_a?(String)
           location.fetch(:ibex_cst_text)
         elsif value.is_a?(String)
           value
         else
           ""
         end
  text = "" if token_id == Parser::EOF_TOKEN
  cache.intern_token_fields(kind: token_id, text: text, leading: leading)
end

#scanned_green_trivia(location, key) ⇒ Array[CST::GreenTrivia]

RBS:

  • (Object? location, Symbol key) -> Array[CST::GreenTrivia]

Parameters:

  • location (Object, nil)
  • key (Symbol)

Returns:



213
214
215
216
217
218
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 213

def scanned_green_trivia(location, key)
  return [] unless location.is_a?(Hash)

  value = location[key]
  value.is_a?(Array) ? value.grep(CST::GreenTrivia) : []
end

#select_lexer_match(input) ⇒ [ Hash[Symbol, untyped]?, String? ]

rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

RBS:

  • (LexerInput input) -> [Hash[Symbol, untyped]?, String?]

Parameters:

Returns:

  • ([ Hash[Symbol, untyped]?, String? ]) 


228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 228

def select_lexer_match(input)
  loop do
    matches = lexer_rules.filter_map do |rule|
      match = rule.fetch(:regexp).match(input.buffer)
      [rule, match[0]] if match&.begin(0)&.zero?
    end #: Array[[Hash[Symbol, untyped], String]]
    boundary_match = matches.any? { |_rule, lexeme| lexeme.length == input.buffer.length }
    if !input.eof? && (matches.empty? || boundary_match)
      input.read_more?
      next
    end

    selected = matches.max_by { |rule, lexeme| [lexeme.bytesize, -rule.fetch(:id)] }
    return [nil, nil] unless selected

    return selected
  end
end

#skip ⇒ nil

Suppress the current match from a lexer action.

RBS:

  • () -> nil

Returns:

  • (nil) 


329
330
331
332
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 329

def skip
  @lexer_skip_requested = true
  nil
end

#split_balanced_trivia(trivia) ⇒ [ Array[CST::GreenTrivia], Array[CST::GreenTrivia] ]

RBS:

  • (Array[CST::GreenTrivia] trivia) -> [Array[CST::GreenTrivia], Array[CST::GreenTrivia]]

Parameters:

Returns:



410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 410

def split_balanced_trivia(trivia)
  return [trivia.freeze, EMPTY_GREEN_TRIVIA] unless trivia.any? { |item| item.text.include?("\n".b) }

  before = [] #: Array[CST::GreenTrivia]
  after = [] #: Array[CST::GreenTrivia]
  found = false
  trivia.each do |item|
    if found
      after << item
      next
    end
    newline = item.text.index("\n".b)
    unless newline
      before << item
      next
    end

    boundary = newline + 1
    before_text = item.text.byteslice(0, boundary) || "".b
    after_text = item.text.byteslice(boundary, item.text.bytesize - boundary) || "".b
    before << CST::GreenTrivia.new(kind: cst_trivia_kind(before_text), text: before_text)
    after << CST::GreenTrivia.new(kind: cst_trivia_kind(after_text), text: after_text) unless after_text.empty?
    found = true
  end
  [before, after]
end

#take_cst_pending_green_trivia ⇒ Array[CST::GreenTrivia]

RBS:

  • () -> Array[CST::GreenTrivia]

Returns:



444
445
446
447
448
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 444

def take_cst_pending_green_trivia
  values = @lexer_pending_green_trivia
  @lexer_pending_green_trivia = []
  values.freeze
end

#validate_lexer_state(state) ⇒ String

RBS:

  • (Symbol | String state) -> String

Parameters:

  • state (Symbol, String)

Returns:

  • (String) 


475
476
477
478
479
480
481
 
# File 'lib/ibex/runtime/generated_lexer.rb', line 475

def validate_lexer_state(state)
  name = state.to_s
  known = self.class.const_get(:LEXER_STATES)
  raise ArgumentError, "unknown lexer state #{state.inspect}" unless known.include?(name)

  name
end