Module: Flexr::Runtime

Included in:
Lexer
Defined in:
lib/flexr/runtime/core.rb,
lib/flexr/runtime/token.rb,
lib/flexr/runtime/buffer.rb,
lib/flexr/runtime/errors.rb,
lib/flexr/runtime/location.rb,
lib/flexr/runtime/interpreter.rb

Defined Under Namespace

Classes: Buffer, BufferTooLargeError, CancelledError, Interpreter, InvalidRecoveryActionError, Location, LookaheadTooLargeError, Match, NonProgressError, ResourceLimitError, StateStackOverflowError, StepLimitError, Token, TokenTooLargeError

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#buffer ⇒ Object (readonly)

Returns the value of attribute buffer.



68
69
70
 
# File 'lib/flexr/runtime/core.rb', line 68

def buffer
  @buffer
end

#error_mode ⇒ Object (readonly)

Returns the value of attribute error_mode.



68
69
70
 
# File 'lib/flexr/runtime/core.rb', line 68

def error_mode
  @error_mode
end

#filename ⇒ Object (readonly)

Returns the value of attribute filename.



68
69
70
 
# File 'lib/flexr/runtime/core.rb', line 68

def filename
  @filename
end

#max_lookahead_size ⇒ Object (readonly)

Returns the value of attribute max_lookahead_size.



68
69
70
 
# File 'lib/flexr/runtime/core.rb', line 68

def max_lookahead_size
  @max_lookahead_size
end

#max_token_size ⇒ Object (readonly)

Returns the value of attribute max_token_size.



68
69
70
 
# File 'lib/flexr/runtime/core.rb', line 68

def max_token_size
  @max_token_size
end

#on_error=(value) ⇒ Object (writeonly)

Sets the attribute on_error

Parameters:

  • value —

    the value to set the attribute on_error to.



70
71
72
 
# File 'lib/flexr/runtime/core.rb', line 70

def on_error=(value)
  @on_error = value
end

#steps ⇒ Object (readonly)

Returns the value of attribute steps.



68
69
70
 
# File 'lib/flexr/runtime/core.rb', line 68

def steps
  @steps
end

Instance Method Details

#begin_state(name) ⇒ Object Also known as: state= 



341
342
343
344
 
# File 'lib/flexr/runtime/core.rb', line 341

def begin_state(name)
  ensure_state!(name)
  @state = name.to_sym
end

#beginning_of_line? ⇒ Boolean

Returns:

  • (Boolean) 


258
259
260
 
# File 'lib/flexr/runtime/core.rb', line 258

def beginning_of_line?
  @bol
end

#binary_input ⇒ Object 



262
263
264
265
 
# File 'lib/flexr/runtime/core.rb', line 262

def binary_input
  source = @buffer.source
  source.encoding == ::Encoding::BINARY ? source : source.b
end

#byte_pos ⇒ Object 



186
187
188
 
# File 'lib/flexr/runtime/core.rb', line 186

def byte_pos
  @position
end

#consume_step!(count = 1) ⇒ Object

Raises:



212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
 
# File 'lib/flexr/runtime/core.rb', line 212

def consume_step!(count = 1)
  return unless scan_steps_guarded?

  @steps += count
  if @max_steps && @steps > @max_steps
    raise Runtime::StepLimitError.new(
      filename: @filename, byte_pos: @position, line: @line, rule: rule_index(@active_rule)
    )
  end
  return unless cancelled?

  raise Runtime::CancelledError.new(
    filename: @filename, byte_pos: @position, line: @line, rule: rule_index(@active_rule)
  )
end

#defer_token_size_check!(size, rule: nil) ⇒ Object

Raises:



194
195
196
197
198
199
200
201
 
# File 'lib/flexr/runtime/core.rb', line 194

def defer_token_size_check!(size, rule: nil)
  return if size <= @max_token_size

  raise Runtime::TokenTooLargeError.new(
    filename: @filename, byte_pos: @more_start || @position, line: @line,
    rule: rule_index(rule || @active_rule)
  )
end

#each_token ⇒ Object 



147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/flexr/runtime/core.rb', line 147

def each_token
  return enum_for(__method__) unless block_given?

  loop do
    token = next_token
    break unless token

    if @token_kind == :yield
      yield(*token)
    else
      yield token
    end
  end
  self
end

#echo ⇒ Object 



307
308
309
 
# File 'lib/flexr/runtime/core.rb', line 307

def echo
  emit(nil, text)
end

#emit(type, value = text) ⇒ Object 



267
268
269
270
271
272
273
274
 
# File 'lib/flexr/runtime/core.rb', line 267

def emit(type, value = text)
  @pending = case @token_kind
  when :struct
    Runtime::Token.new(type: type, value: value, location: last_location)
  else
    [type, value]
  end
end

#ensure_lookahead_size!(size, rule: nil) ⇒ Object

Raises:



203
204
205
206
207
208
209
210
 
# File 'lib/flexr/runtime/core.rb', line 203

def ensure_lookahead_size!(size, rule: nil)
  return if size <= @max_lookahead_size

  raise Runtime::LookaheadTooLargeError.new(
    filename: @filename, byte_pos: @position, line: @line,
    rule: rule_index(rule || @active_rule)
  )
end

#error!(message) ⇒ Object 



280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
 
# File 'lib/flexr/runtime/core.rb', line 280

def error!(message)
  error = LexError.new(message, filename: @filename, byte_pos: @match_start, line: @text_line, text: text)
  if @on_error
    action = @on_error.call(error)
    return @pending = nil if action == :skip
    raise error if action == :raise
    if action == :halt
      @halted = true
      return @pending = nil
    end
    return emit(:error, text) if action == :token

    raise Runtime::InvalidRecoveryActionError.new(
      action: action, filename: @filename, byte_pos: @match_start, line: @text_line,
      text: text, rule: rule_index(@active_rule)
    )
  end
  case @error_mode
  when :token
    emit(:error, text)
  when :panic
    @pending = nil
  else
    raise error
  end
end

#initialize(input, filename: nil, error_mode: :raise, max_token_size: 16 * 1024 * 1024, max_lookahead_size: nil, max_buffer_size: 64 * 1024 * 1024, max_state_stack: 1024, max_steps: nil, cancellation: nil, retain_input: true, chunk_size: Runtime::Buffer::DEFAULT_CHUNK_SIZE) ⇒ Object

Raises:

  • (ArgumentError) 


5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/flexr/runtime/core.rb', line 5

def initialize(input, filename: nil, error_mode: :raise, max_token_size: 16 * 1024 * 1024,
               max_lookahead_size: nil, max_buffer_size: 64 * 1024 * 1024,
               max_state_stack: 1024, max_steps: nil, cancellation: nil, retain_input: true,
               chunk_size: Runtime::Buffer::DEFAULT_CHUNK_SIZE)
  raise ArgumentError, "max_token_size must be non-negative" if max_token_size.to_i.negative?
  raise ArgumentError, "max_lookahead_size must be non-negative" if max_lookahead_size&.to_i&.negative?
  raise ArgumentError, "max_buffer_size must be non-negative" if max_buffer_size.to_i.negative?
  raise ArgumentError, "max_state_stack must be non-negative" if max_state_stack.to_i.negative?
  raise ArgumentError, "max_steps must be non-negative" if max_steps&.to_i&.negative?
  raise ArgumentError, "cancellation must respond to call" if cancellation && !cancellation.respond_to?(:call)

  self.class.compile!
  config = self.class.__flexr_config
  @buffer = Runtime::Buffer.new(
    input, chunk_size: chunk_size, max_buffer_size: max_buffer_size,
    retain_input: retain_input, filename: filename
  )
  @string_input = input.is_a?(String)
  @stable_input_end = input.bytesize if @string_input && input.frozen?
  @valid_utf8_input = @string_input && input.frozen? && (config.encoding != Encoding::UTF_8 || begin
    bytes = input.encoding == Encoding::UTF_8 ? input : input.dup.force_encoding(Encoding::UTF_8)
    bytes.valid_encoding?
  end)
  @simple_location_input = @string_input && input.frozen? && retain_input &&
    input.ascii_only? && !input.include?("\n")
  @retain_input = retain_input
  @filename = filename
  @error_mode = error_mode
  @token_kind = config.token_kind
  @accel_mode = config.options.fetch(:accel, :auto)
  @max_token_size = max_token_size.to_i
  @token_limit_required = !@string_input || !input.frozen? || input.bytesize > @max_token_size
  @max_lookahead_size = (max_lookahead_size || max_token_size).to_i
  @max_state_stack = max_state_stack.to_i
  @max_steps = max_steps&.to_i
  @cancellation = cancellation
  @steps = 0
  @position = 0
  @line = 1
  @column = 1
  @state = :initial
  @state_stack = []
  @pending = nil
  @matched = nil
  @match_start = 0
  @match_end = 0
  @text_start = 0
  @text_line = 1
  @text_column = 1
  @bol = true
  @more_start = nil
  @more_line = nil
  @more_column = nil
  @more_requested = false
  @non_progress_signatures = {}
  @active_rule = nil
  @eof_fired_states = {}
  @on_error = nil
  @halted = false
  @interpreter = nil
  @generated_scanner = generated_runtime? && respond_to?(:scan_one, true)
end

#input ⇒ Object 



72
73
74
 
# File 'lib/flexr/runtime/core.rb', line 72

def input
  @buffer.source
end

#last_location ⇒ Object 



365
366
367
368
369
370
371
372
373
374
375
376
377
 
# File 'lib/flexr/runtime/core.rb', line 365

def last_location
  line_begin = @text_line || @line
  column_begin = @text_column || @column
  line_end, column_end = location_after(
    @text_start, @match_end, line: line_begin, column: column_begin
  )
  Runtime::Location.new(
    filename: @filename, byte_begin: @text_start, byte_end: @match_end,
    line_begin: line_begin, line_end: line_end,
    column_values: [column_begin, column_end],
    eager_columns: self.class.__flexr_config.options[:eager_columns] == true
  )
end

#less(count) ⇒ Object

Raises:

  • (ArgumentError) 


348
349
350
351
352
353
354
355
356
357
358
359
 
# File 'lib/flexr/runtime/core.rb', line 348

def less(count)
  matched_bytes = @match_end - @match_start
  raise ArgumentError, "less must not exceed matched bytes" if count.negative? || count > matched_bytes

  new_position = @match_start + count
  raise ArgumentError, "less must end at a UTF-8 codepoint boundary" if
    utf8_input? && !@buffer.utf8_boundary?(new_position)

  @position = new_position
  @match_end = @position
  @matched = nil
end

#lineno ⇒ Object Also known as: line 



248
249
250
 
# File 'lib/flexr/runtime/core.rb', line 248

def lineno
  @line
end

#more ⇒ Object 



361
362
363
 
# File 'lib/flexr/runtime/core.rb', line 361

def more
  @more_requested = true
end

#more_text_start ⇒ Object 



190
191
192
 
# File 'lib/flexr/runtime/core.rb', line 190

def more_text_start
  @more_start
end

#next_token ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
 
# File 'lib/flexr/runtime/core.rb', line 76

def next_token
  return nil if @halted

  loop do
    return nil if @halted
    consume_step! if scan_steps_guarded?

    if eof? && @pending.nil?
      eof_action = self.class.__flexr_spec.eof_rules[@state]
      if eof_action && !@eof_fired_states[@state]
        @eof_fired_states[@state] = true
        @match_start = @position
        @match_end = @position
        @text_start = @position
        @text_line = @line
        @text_column = @column
        @matched = nil
        instance_exec(&eof_action)
        token = @pending
        @pending = nil
        return token if token
        next
      end
      return nil
    end

    @active_rule = nil
    match = if @generated_scanner
      scan_one
    else
      (@interpreter ||= Runtime::Interpreter.new(self)).scan
    end
    unless match
      unless eof?
        token = handle_unmatched_byte
        next unless token

        return token
      end
      return nil
    end
    @match_start = match.start_pos
    @match_end = match.end_pos
    scan_state = @state
    if @more_start
      @text_start = @more_start
      @text_line = @more_line
      @text_column = @more_column
    else
      @text_start = @match_start
      @text_line = @line
      @text_column = @column
    end
    @matched = nil
    @more_requested = false
    @position = match.end_pos
    empty_match = match.end_pos == match.start_pos
    @active_rule = match.rule
    execute(match.rule)
    finalize_more
    ensure_progress!(match, scan_state, empty_match: empty_match) unless
      @position > match.start_pos && @non_progress_signatures.empty?
    ensure_token_size! if @token_limit_required
    update_position
    discard_consumed_input! unless @retain_input
    token = @pending
    @pending = nil
    return token if token
  end
end

#pop ⇒ Object Also known as: pop_state 



335
336
337
 
# File 'lib/flexr/runtime/core.rb', line 335

def pop
  @state = @state_stack.pop || :initial
end

#push(name) ⇒ Object Also known as: push_state 



319
320
321
322
323
324
325
326
327
328
329
330
331
 
# File 'lib/flexr/runtime/core.rb', line 319

def push(name)
  ensure_state!(name)
  if @state_stack.length >= @max_state_stack
    raise Runtime::StateStackOverflowError.new(
      "state stack exceeds max_state_stack (#{@max_state_stack})",
      filename: @filename, byte_pos: @position, line: @line, text: text,
      rule: rule_index(@active_rule)
    )
  end

  @state_stack << @state
  @state = name.to_sym
end

#racc_next_token ⇒ Object 



171
172
173
174
 
# File 'lib/flexr/runtime/core.rb', line 171

def racc_next_token
  token = next_token
  token ? [token[0], token[1]] : [false, "$end"]
end

#reject ⇒ Object

Raises:



311
312
313
314
315
316
317
 
# File 'lib/flexr/runtime/core.rb', line 311

def reject
  diagnostic = Diagnostics.error(
    "FLEXR-E013", "reject is not supported by flexr",
    help: "use a state transition and less(n) to express the fallback"
  )
  raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
end

#scan_steps_guarded? ⇒ Boolean

Returns:

  • (Boolean) 


228
229
230
 
# File 'lib/flexr/runtime/core.rb', line 228

def scan_steps_guarded?
  !@max_steps.nil? || !@cancellation.nil?
end

#skip ⇒ Object 



276
277
278
 
# File 'lib/flexr/runtime/core.rb', line 276

def skip
  @pending = nil
end

#state ⇒ Object 



254
255
256
 
# File 'lib/flexr/runtime/core.rb', line 254

def state
  @state
end

#text ⇒ Object 



176
177
178
179
180
 
# File 'lib/flexr/runtime/core.rb', line 176

def text
  return @matched if @matched

  @matched = @buffer.byteslice(@text_start...@match_end)
end

#text_bytesize ⇒ Object 



182
183
184
 
# File 'lib/flexr/runtime/core.rb', line 182

def text_bytesize
  @match_end - @text_start
end

#token_limit_required? ⇒ Boolean

Returns:

  • (Boolean) 


232
233
234
 
# File 'lib/flexr/runtime/core.rb', line 232

def token_limit_required?
  @token_limit_required
end

#tokens ⇒ Object 



163
164
165
166
167
168
169
 
# File 'lib/flexr/runtime/core.rb', line 163

def tokens
  result = []
  while (token = next_token)
    result << token
  end
  result
end

#utf8_boundary?(position) ⇒ Boolean

Returns:

  • (Boolean) 


244
245
246
 
# File 'lib/flexr/runtime/core.rb', line 244

def utf8_boundary?(position)
  !utf8_input? || @buffer.utf8_boundary?(position)
end

#utf8_input? ⇒ Boolean

Returns:

  • (Boolean) 


236
237
238
 
# File 'lib/flexr/runtime/core.rb', line 236

def utf8_input?
  self.class.__flexr_config.encoding == Encoding::UTF_8
end

#valid_utf8_at?(position) ⇒ Boolean

Returns:

  • (Boolean) 


240
241
242
 
# File 'lib/flexr/runtime/core.rb', line 240

def valid_utf8_at?(position)
  @valid_utf8_input || !utf8_input? || @buffer.valid_utf8_at?(position)
end