Class: Ibex::BisonImport::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/ibex/bison_import/tokenizer.rb,
sig/ibex/bison_import/tokenizer.rbs

Overview

Iterative scanner for the punctuation needed to recover Bison rules.

Defined Under Namespace

Classes: Token

Constant Summary collapse

IDENTIFIER_START =

Signature:

  • Regexp

Returns:

  • (Regexp) 
/[A-Za-z_$.]/
IDENTIFIER_CONTINUE =

Signature:

  • Regexp

Returns:

  • (Regexp) 
/[A-Za-z0-9_$.-]/

Instance Method Summary collapse

Constructor Details

#initialize(source, start_line:, max_tokens:) ⇒ Tokenizer

Returns a new instance of Tokenizer.

RBS:

  • (String source, start_line: Integer, max_tokens: Integer) -> void

Parameters:

  • source (String)
  • start_line: (Integer)
  • max_tokens: (Integer) 


29
30
31
32
33
34
35
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 29

def initialize(source, start_line:, max_tokens:)
  @source = source
  @index = 0
  @line = start_line
  @column = 1
  @max_tokens = max_tokens
end

Instance Method Details

#advance ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



247
248
249
250
251
252
253
254
255
256
257
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 247

def advance
  return if eof?

  if current_byte == 10
    @line += 1
    @column = 1
  else
    @column += 1
  end
  @index += 1
end

#check_token_budget(count) ⇒ void

This method returns an undefined value.

RBS:

  • (Integer count) -> void

Parameters:

  • count (Integer) 


223
224
225
226
227
228
229
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 223

def check_token_budget(count)
  return if count <= @max_tokens

  raise BudgetExceeded.new(
    result: "budget_exhausted", phase: "tokenization", max_tokens: @max_tokens
  )
end

#current_byte ⇒ Integer?

RBS:

  • () -> Integer?

Returns:

  • (Integer, nil) 


232
233
234
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 232

def current_byte
  @source.getbyte(@index)
end

#eof? ⇒ Boolean

RBS:

  • () -> bool

Returns:

  • (Boolean) 


242
243
244
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 242

def eof?
  @index >= @source.bytesize
end

#identifier_continue?(byte) ⇒ Boolean

RBS:

  • (Integer? byte) -> bool

Parameters:

  • byte (Integer, nil)

Returns:

  • (Boolean) 


213
214
215
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 213

def identifier_continue?(byte)
  byte ? IDENTIFIER_CONTINUE.match?(byte.chr) : false
end

#identifier_start?(byte) ⇒ Boolean

RBS:

  • (Integer? byte) -> bool

Parameters:

  • byte (Integer, nil)

Returns:

  • (Boolean) 


208
209
210
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 208

def identifier_start?(byte)
  byte ? IDENTIFIER_START.match?(byte.chr) : false
end

#peek_byte ⇒ Integer?

RBS:

  • () -> Integer?

Returns:

  • (Integer, nil) 


237
238
239
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 237

def peek_byte
  @source.getbyte(@index + 1)
end

#punctuation_type(value) ⇒ Symbol

RBS:

  • (String value) -> Symbol

Parameters:

  • value (String)

Returns:

  • (Symbol) 


198
199
200
201
202
203
204
205
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 198

def punctuation_type(value)
  case value
  when ":" then :colon
  when "|" then :pipe
  when ";" then :semicolon
  else :other
  end
end

#scan_action ⇒ String

RBS:

  • () -> String

Returns:

  • (String) 


123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 123

def scan_action
  advance
  start = @index
  depth = 1
  until eof?
    byte = current_byte
    if skip_action_opaque?(byte)
      next
    elsif byte == 123
      depth += 1
      advance
    elsif byte == 125
      depth -= 1
      if depth.zero?
        value = @source.byteslice(start, @index - start) || ""
        advance
        return value
      end
      advance
    else
      advance
    end
  end
  raise Ibex::Error, "(bison-import):#{@line}:#{@column}: unterminated C action"
end

#scan_delimited(opener, closer) ⇒ String

RBS:

  • (Integer opener, Integer closer) -> String

Parameters:

  • opener (Integer)
  • closer (Integer)

Returns:

  • (String) 


164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 164

def scan_delimited(opener, closer)
  start = @index
  advance
  until eof? || current_byte == closer
    current_byte == 92 ? 2.times { advance unless eof? } : advance
  end
  advance if current_byte == closer
  value = @source.byteslice(start, @index - start) || ""
  return value if value.getbyte(0) == opener

  ""
end

#scan_directive ⇒ String

RBS:

  • () -> String

Returns:

  • (String) 


98
99
100
101
102
103
104
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 98

def scan_directive
  start = @index
  advance
  advance if current_byte == 37
  advance while identifier_continue?(current_byte)
  @source.byteslice(start, @index - start) || ""
end

#scan_identifier ⇒ String

RBS:

  • () -> String

Returns:

  • (String) 


91
92
93
94
95
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 91

def scan_identifier
  start = @index
  advance while identifier_continue?(current_byte)
  @source.byteslice(start, @index - start) || ""
end

#scan_quoted(quote) ⇒ String

RBS:

  • (Integer quote) -> String

Parameters:

  • quote (Integer)

Returns:

  • (String) 


107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 107

def scan_quoted(quote)
  start = @index
  advance
  until eof?
    byte = current_byte
    advance
    if byte == 92
      advance unless eof?
    elsif byte == quote
      break
    end
  end
  @source.byteslice(start, @index - start) || ""
end

#scan_token ⇒ Token

RBS:

  • () -> Token

Returns:



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 53

def scan_token
  line = @line
  column = @column
  byte = current_byte || raise(Ibex::Error, "(bison-import):#{line}:#{column}: unexpected end of input")
  if identifier_start?(byte)
    Token.new(type: :symbol, value: scan_identifier, line: line, column: column)
  elsif [34, 39].include?(byte)
    Token.new(type: :literal, value: scan_quoted(byte), line: line, column: column)
  elsif byte == 123
    Token.new(type: :action, value: scan_action, line: line, column: column)
  elsif byte == 37
    Token.new(type: :directive, value: scan_directive, line: line, column: column)
  elsif byte == 60
    Token.new(type: :tag, value: scan_delimited(60, 62), line: line, column: column)
  elsif byte == 91
    Token.new(type: :tag, value: scan_delimited(91, 93), line: line, column: column)
  else
    value = byte ? byte.chr : ""
    advance
    Token.new(type: punctuation_type(value), value: value, line: line, column: column)
  end
end

#skip_action_opaque?(byte) ⇒ Boolean

RBS:

  • (Integer? byte) -> bool

Parameters:

  • byte (Integer, nil)

Returns:

  • (Boolean) 


150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 150

def skip_action_opaque?(byte)
  if byte && [34, 39].include?(byte)
    scan_quoted(byte)
  elsif byte == 47 && peek_byte == 42
    skip_block_comment
  elsif byte == 47 && peek_byte == 47
    skip_line
  else
    return false
  end
  true
end

#skip_block_comment ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



178
179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 178

def skip_block_comment
  advance
  advance
  until eof?
    if current_byte == 42 && peek_byte == 47
      advance
      advance
      return
    end
    advance
  end
end

#skip_ignored ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 77

def skip_ignored
  loop do
    advance while whitespace?(current_byte)
    if current_byte == 47 && peek_byte == 42
      skip_block_comment
    elsif (current_byte == 47 && peek_byte == 47) || current_byte == 35
      skip_line
    else
      break
    end
  end
end

#skip_line ⇒ void

This method returns an undefined value.

RBS:

  • () -> void



192
193
194
195
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 192

def skip_line
  advance until eof? || current_byte == 10
  advance if current_byte == 10
end

#tokenize ⇒ Array[Token]

RBS:

  • () -> Array[Token]

Returns:



38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 38

def tokenize
  tokens = [] #: Array[Token]
  until eof?
    skip_ignored
    break if eof?

    tokens << scan_token
    check_token_budget(tokens.length)
  end
  tokens.freeze
end

#whitespace?(byte) ⇒ Boolean

RBS:

  • (Integer? byte) -> bool

Parameters:

  • byte (Integer, nil)

Returns:

  • (Boolean) 


218
219
220
 
# File 'lib/ibex/bison_import/tokenizer.rb', line 218

def whitespace?(byte)
  !byte.nil? && [9, 10, 11, 12, 13, 32].include?(byte)
end