Class: Ibex::Frontend::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/ibex/frontend/lexer.rb,
lib/ibex/frontend/lexer_recovery.rb,
sig/ibex/frontend/lexer.rbs,
sig/ibex/frontend/lexer_recovery.rbs

Overview

Tokenizes a racc-compatible grammar while preserving source locations. rubocop:disable Metrics/ClassLength -- token scanners stay together to preserve cursor invariants.

Constant Summary collapse

PUNCTUATION =

Returns:

  • (Hash[String, Symbol]) 
%w[: | ; = < > ? * + , ( )].to_h do |character|
  [character, character.to_sym]
end.freeze
USER_CODE =

Signature:

  • Hash[String, Symbol]

Returns:

  • (Regexp) 
/\A----[ \t]+(header|inner|footer)[ \t]*(?:\r?\n|\z)/
NODE_DIRECTIVE =

Signature:

  • Regexp

Returns:

  • (Regexp) 
/\A@node(?=\s|\z)/
PERCENT_DIRECTIVES =

Returns:

  • (Hash[String, [ Symbol, String ]]) 
{
  "%expect-rr" => [:identifier, "expect_rr"],
  "%precedence" => [:identifier, "precedence"],
  "%param" => [:identifier, "param"], "%printer" => [:identifier, "printer"],
  "%recover" => [:identifier, "recover"], "%on_error_reduce" => [:identifier, "on_error_reduce"],
  "%test" => [:identifier, "test"], "%inline" => [:inline, "%inline"],
  "%empty" => [:empty, "%empty"]
}.freeze

Instance Method Summary collapse

Constructor Details

#initialize(source, file: "(grammar)") ⇒ Lexer

Returns a new instance of Lexer.

RBS:

  • (String source, ?file: String) -> void

Parameters:

  • source (String)
  • file: (String) (defaults to: "(grammar)") 


29
30
31
32
33
 
# File 'lib/ibex/frontend/lexer.rb', line 29

def initialize(source, file: "(grammar)")
  @cursor = SourceCursor.new(source, file)
  @segments = [] #: Array[Segment]
  @emitted_tokens = [] #: Array[Token]
end

Instance Method Details

#advance_after_lexical_error(error, start) ⇒ void

This method returns an undefined value.

RBS:

  • (Exception error, SourcePosition start) -> void

Parameters:



75
76
77
78
79
80
81
 
# File 'lib/ibex/frontend/lexer_recovery.rb', line 75

def advance_after_lexical_error(error, start)
  return unless @cursor.position.byte_offset == start.byte_offset

  remaining = @cursor.source.length - @cursor.index
  count = error.message.include?("unterminated block comment") ? remaining : 1
  @cursor.advance(count)
end

#emit_invalid_segment(start) ⇒ void

This method returns an undefined value.

RBS:

  • (SourcePosition start) -> void

Parameters:



61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/ibex/frontend/lexer_recovery.rb', line 61

def emit_invalid_segment(start)
  span = @cursor.span_from(start)
  previous = @segments.last
  unless previous&.kind == :invalid && previous.span.end_byte == span.start_byte
    emit_segment(:invalid, start)
    return
  end

  combined_span = SourceSpan.new(file: @cursor.file, start: previous.span.start, finish: span.finish)
  text = @cursor.source.byteslice(combined_span.start_byte, combined_span.length) || ""
  @segments[-1] = Segment.new(kind: :invalid, span: combined_span, text: text)
end

#emit_segment(kind, start, token_type: nil, token_index: nil) ⇒ void

This method returns an undefined value.

RBS:

  • (Symbol kind, SourcePosition start, ?token_type: Symbol?, ?token_index: Integer?) -> void

Parameters:

  • kind (Symbol)
  • start (SourcePosition)
  • token_type: (Symbol, nil) (defaults to: nil)
  • token_index: (Integer, nil) (defaults to: nil) 


308
309
310
311
312
313
 
# File 'lib/ibex/frontend/lexer.rb', line 308

def emit_segment(kind, start, token_type: nil, token_index: nil)
  span = @cursor.span_from(start)
  text = @cursor.source.byteslice(span.start_byte, span.length) || ""
  @segments << Segment.new(kind: kind, span: span, text: text,
                           token_type: token_type, token_index: token_index)
end

#emit_token(token, start) ⇒ void

This method returns an undefined value.

RBS:

  • (Token token, SourcePosition start) -> void

Parameters:



297
298
299
300
301
302
303
304
305
 
# File 'lib/ibex/frontend/lexer.rb', line 297

def emit_token(token, start)
  token.span = @cursor.span_from(start)
  kind = case token.type
         when :action then :action
         when :eof then :eof
         else :token
         end
  emit_segment(kind, start, token_type: token.type, token_index: @emitted_tokens.length)
end

#lexical_error_details(error, fallback) ⇒ [ Location, String ]

RBS:

  • (Exception error, SourcePosition fallback) -> [Location, String]

Parameters:

Returns:



46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/ibex/frontend/lexer_recovery.rb', line 46

def lexical_error_details(error, fallback)
  pattern = /\A#{Regexp.escape(@cursor.file)}:(\d+):(\d+): (.*)\z/m
  match = error.message.match(pattern)
  if match
    return [
      Location.new(file: @cursor.file, line: Integer(match[1]), column: Integer(match[2])),
      match[3] || ""
    ]
  end

  location = Location.new(file: @cursor.file, line: fallback.line, column: fallback.column)
  [location, error.message.delete_prefix("#{location}: ")]
end

#line_start? ⇒ Boolean

RBS:

  • () -> bool

Returns:

  • (Boolean) 


154
155
156
 
# File 'lib/ibex/frontend/lexer.rb', line 154

def line_start?
  @cursor.column == 1
end

#newline? ⇒ Boolean

RBS:

  • () -> bool

Returns:

  • (Boolean) 


124
125
126
 
# File 'lib/ibex/frontend/lexer.rb', line 124

def newline?
  @cursor.peek == "\n" || @cursor.rest.start_with?("\r\n")
end

#next_token ⇒ Token

RBS:

  • () -> Token

Returns:



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/ibex/frontend/lexer.rb', line 53

def next_token
  eof_token = @eof_token
  return eof_token if eof_token

  @active_token_start = nil
  skip_ignored
  start = @cursor.position
  @active_token_start = start
  scanned = if @cursor.eof?
              token(:eof, nil)
            else
              scan_from_cursor
            end
  emit_token(scanned, start) unless scanned.span
  @emitted_tokens << scanned
  @eof_token = scanned if scanned.type == :eof
  @active_token_start = nil
  scanned
end

#recover_lexical_error(error, capture:) ⇒ Diagnostic?

RBS:

  • (Exception error, capture: bool) -> Diagnostic?

Parameters:

  • error (Exception)
  • capture: (Boolean)

Returns:



31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/ibex/frontend/lexer_recovery.rb', line 31

def recover_lexical_error(error, capture:)
  start = @active_token_start || @cursor.position
  advance_after_lexical_error(error, start)
  emit_invalid_segment(start)
  return unless capture

  location, message = lexical_error_details(error, start)
  Diagnostic.new(code: "frontend.lexical_error", phase: :lexical,
                 message: message, location: location,
                 span: @cursor.span_from(start), rendered: error.message)
ensure
  @active_token_start = nil
end

#scan_from_cursor ⇒ Token

RBS:

  • () -> Token

Returns:



76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/ibex/frontend/lexer.rb', line 76

def scan_from_cursor
  character = @cursor.peek || ""

  return scan_user_code if line_start? && @cursor.rest.start_with?("----")
  return ActionScanner.new(@cursor).scan if character == "{"
  return scan_scope if @cursor.rest.start_with?("::")
  return scan_percent_directive if @cursor.peek == "%"
  return scan_node_directive if @cursor.rest.match?(NODE_DIRECTIVE)

  scan_regular_token(character)
end

#scan_identifier ⇒ Token

RBS:

  • () -> Token

Returns:



210
211
212
 
# File 'lib/ibex/frontend/lexer.rb', line 210

def scan_identifier
  scan_match(:identifier, /\A[A-Za-z_][A-Za-z0-9_]*/)
end

#scan_integer ⇒ Token

RBS:

  • () -> Token

Returns:



215
216
217
 
# File 'lib/ibex/frontend/lexer.rb', line 215

def scan_integer
  scan_match(:integer, /\A\d+/) { |value| Integer(value, 10) }
end

#scan_line_comment ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



136
137
138
139
140
 
# File 'lib/ibex/frontend/lexer.rb', line 136

def scan_line_comment
  start = @cursor.position
  @cursor.advance until @cursor.eof? || newline?
  emit_segment(:line_comment, start)
end

#scan_literal ⇒ Token

RBS:

  • () -> Token

Returns:



220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
 
# File 'lib/ibex/frontend/lexer.rb', line 220

def scan_literal
  location = @cursor.location
  quote = @cursor.peek
  start = @cursor.index
  @cursor.advance
  until @cursor.eof?
    if @cursor.peek == "\\"
      @cursor.advance(2)
    elsif @cursor.peek == quote
      @cursor.advance
      value = @cursor.source[start...@cursor.index] || ""
      return token(:literal, value, location)
    else
      @cursor.advance
    end
  end
  raise Ibex::Error, "#{location}: unterminated quoted token"
end

#scan_match(type, pattern) ⇒ void

This method returns an undefined value.

RBS:

  • (Symbol type, Regexp pattern) ?{ (String) -> token_value } -> Token

Parameters:

  • type (Symbol)
  • pattern (Regexp) 


281
282
283
284
285
286
287
288
289
 
# File 'lib/ibex/frontend/lexer.rb', line 281

def scan_match(type, pattern)
  location = @cursor.location
  value = @cursor.rest.match(pattern)&.[](0)
  raise Ibex::Error, "#{location}: invalid #{type} token" unless value

  @cursor.advance(value.length)
  value = yield(value) if block_given?
  token(type, value, location)
end

#scan_newline ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



129
130
131
132
133
 
# File 'lib/ibex/frontend/lexer.rb', line 129

def scan_newline
  start = @cursor.position
  @cursor.advance(@cursor.rest.start_with?("\r\n") ? 2 : 1)
  emit_segment(:newline, start)
end

#scan_node_directive ⇒ Token

RBS:

  • () -> Token

Returns:



203
204
205
206
207
 
# File 'lib/ibex/frontend/lexer.rb', line 203

def scan_node_directive
  location = @cursor.location
  @cursor.advance("@node".length)
  token(:node, "@node", location)
end

#scan_percent_directive ⇒ Token

RBS:

  • () -> Token

Returns:



190
191
192
193
194
195
196
197
198
199
200
 
# File 'lib/ibex/frontend/lexer.rb', line 190

def scan_percent_directive
  location = @cursor.location
  directive = PERCENT_DIRECTIVES.each_key.find do |candidate|
    @cursor.rest.match?(/\A#{Regexp.escape(candidate)}(?=\s|\z)/)
  end
  raise Ibex::Error, "#{location}: unexpected character \"%\"" unless directive

  type, value = PERCENT_DIRECTIVES.fetch(directive)
  @cursor.advance(directive.length)
  token(type, value, location)
end

#scan_punctuation ⇒ Token

RBS:

  • () -> Token

Returns:



271
272
273
274
275
276
277
278
 
# File 'lib/ibex/frontend/lexer.rb', line 271

def scan_punctuation
  location = @cursor.location
  character = @cursor.peek
  raise Ibex::Error, "#{location}: expected punctuation" unless character

  @cursor.advance
  token(PUNCTUATION.fetch(character), character, location)
end

#scan_regexp ⇒ Token

rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

RBS:

  • () -> Token

Returns:



241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/ibex/frontend/lexer.rb', line 241

def scan_regexp
  location = @cursor.location
  start = @cursor.index
  @cursor.advance
  in_class = false
  until @cursor.eof?
    character = @cursor.peek
    if character == "\\"
      @cursor.advance(2)
    elsif character == "["
      in_class = true
      @cursor.advance
    elsif character == "]" && in_class
      in_class = false
      @cursor.advance
    elsif character == "/" && !in_class
      @cursor.advance
      @cursor.advance while @cursor.peek&.match?(/[imx]/)
      return token(:regexp, @cursor.source[start...@cursor.index] || "", location)
    elsif newline?
      raise Ibex::Error, "#{location}: unterminated regular expression"
    else
      @cursor.advance
    end
  end
  raise Ibex::Error, "#{location}: unterminated regular expression"
end

#scan_regular_token(character) ⇒ Token

RBS:

  • (String character) -> Token

Parameters:

  • character (String)

Returns:



89
90
91
92
93
94
95
96
97
 
# File 'lib/ibex/frontend/lexer.rb', line 89

def scan_regular_token(character)
  return scan_identifier if character.match?(/[A-Za-z_]/)
  return scan_integer if character.match?(/\d/)
  return scan_literal if ["'", '"'].include?(character)
  return scan_regexp if character == "/"
  return scan_punctuation if PUNCTUATION.key?(character)

  raise Ibex::Error, "#{@cursor.location}: unexpected character #{character.inspect}"
end

#scan_scope ⇒ Token

RBS:

  • () -> Token

Returns:



183
184
185
186
187
 
# File 'lib/ibex/frontend/lexer.rb', line 183

def scan_scope
  location = @cursor.location
  @cursor.advance(2)
  token(:scope, "::", location)
end

#scan_user_code ⇒ Token

RBS:

  • () -> Token

Returns:



159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/ibex/frontend/lexer.rb', line 159

def scan_user_code
  start_position = @cursor.position
  location = @cursor.location
  match = @cursor.rest.match(USER_CODE)
  raise Ibex::Error, "#{location}: expected ---- header, inner, or footer" unless match

  name = match[1]
  marker = match[0]
  raise Ibex::Error, "#{location}: expected ---- header, inner, or footer" unless name && marker

  @cursor.advance(marker.length)
  emit_segment(
    :user_code_marker, start_position, token_type: :user_code, token_index: @emitted_tokens.length
  )
  body_position = @cursor.position
  start = @cursor.index
  finish = @cursor.source.index(/^----/, start) || @cursor.source.length
  @cursor.advance(finish - start)
  code = @cursor.source[start...finish] || ""
  emit_segment(:user_code_body, body_position)
  token(:user_code, { name: name, code: code }, location, @cursor.span_from(start_position))
end

#scan_whitespace ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



117
118
119
120
121
 
# File 'lib/ibex/frontend/lexer.rb', line 117

def scan_whitespace
  start = @cursor.position
  @cursor.advance while @cursor.peek&.match?(/\s/) && !newline?
  emit_segment(:whitespace, start)
end

#skip_block_comment ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



143
144
145
146
147
148
149
150
151
 
# File 'lib/ibex/frontend/lexer.rb', line 143

def skip_block_comment
  start = @cursor.position
  location = @cursor.location
  finish = @cursor.source.index("*/", @cursor.index + 2)
  raise Ibex::Error, "#{location}: unterminated block comment" unless finish

  @cursor.advance(finish + 2 - @cursor.index)
  emit_segment(:block_comment, start)
end

#skip_ignored ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/ibex/frontend/lexer.rb', line 100

def skip_ignored
  loop do
    if newline?
      scan_newline
    elsif @cursor.peek&.match?(/\s/)
      scan_whitespace
    elsif @cursor.peek == "#"
      scan_line_comment
    elsif @cursor.rest.start_with?("/*")
      skip_block_comment
    else
      return
    end
  end
end

#token(type, value, location = @cursor.location, span = nil) ⇒ Token

RBS:

  • (Symbol type, token_value value, ?Location location, ?SourceSpan? span) -> Token

Parameters:

  • type (Symbol)
  • value (token_value)
  • location (Location) (defaults to: @cursor.location)
  • span (SourceSpan, nil) (defaults to: nil)

Returns:



292
293
294
 
# File 'lib/ibex/frontend/lexer.rb', line 292

def token(type, value, location = @cursor.location, span = nil)
  Token.new(type: type, value: value, location: location, span: span)
end

#tokenize ⇒ Array[Token]

RBS:

  • () -> Array[Token]

Returns:



36
37
38
39
40
41
42
43
 
# File 'lib/ibex/frontend/lexer.rb', line 36

def tokenize
  tokens = [] #: Array[Token]
  loop do
    token = next_token
    tokens << token
    return tokens if token.type == :eof
  end
end

#tokenize_document ⇒ SourceDocument

RBS:

  • () -> SourceDocument

Returns:



46
47
48
49
50
 
# File 'lib/ibex/frontend/lexer.rb', line 46

def tokenize_document
  tokenize unless @eof_token
  cst = CST::Document.new(@segments)
  SourceDocument.new(source: @cursor.source, file: @cursor.file, tokens: @emitted_tokens, cst: cst)
end

#tokenize_document_recovering(max_diagnostics: 20) ⇒ [ SourceDocument, Array[Diagnostic] ]

Recover lexical failures while retaining every source byte.

RBS:

  • (?max_diagnostics: Integer) -> [SourceDocument, Array[Diagnostic]]

Parameters:

  • max_diagnostics: (Integer) (defaults to: 20)

Returns:



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/ibex/frontend/lexer_recovery.rb', line 8

def tokenize_document_recovering(max_diagnostics: 20)
  unless max_diagnostics.is_a?(Integer) && max_diagnostics.positive?
    raise ArgumentError, "max_diagnostics must be a positive integer"
  end

  diagnostics = [] #: Array[Diagnostic]
  until @eof_token
    begin
      next_token
    rescue Ibex::Error => e
      diagnostic = recover_lexical_error(e, capture: diagnostics.length < max_diagnostics)
      diagnostics << diagnostic if diagnostic
    end
  end
  cst = CST::Document.new(@segments)
  document = SourceDocument.new(source: @cursor.source, file: @cursor.file,
                                tokens: @emitted_tokens, cst: cst)
  [document, diagnostics.freeze]
end