Class: Ibex::Frontend::TokenAdapter::DeclarationState

Inherits:
Object
  • Object
show all
Includes:
DeclarationDocumentState, DeclarationLexerState
Defined in:
lib/ibex/frontend/token_adapter/declaration_state.rb,
sig/ibex/frontend/token_adapter/declaration_state.rbs

Overview

Classifies tokens through the class header and declaration section. rubocop:disable Metrics/ClassLength, Metrics/CyclomaticComplexity One state machine owns declaration-boundary classification.

Constant Summary collapse

DECLARATIONS =

Returns:

  • (Hash[String, [ external_token, Symbol ]]) 
{
  "include" => %i[INCLUDE include_path],
  "import" => %i[IMPORT include_path],
  "token" => %i[TOKEN token_symbols], "options" => %i[OPTIONS options_identifiers],
  "expect" => %i[EXPECT expect_integer], "start" => %i[START start_first_symbol],
  "expect_rr" => %i[EXPECT_RR expect_rr_integer],
  "recover" => %i[RECOVER recover_kind],
  "on_error_reduce" => %i[ON_ERROR_REDUCE on_error_reduce_first_symbol],
  "test" => %i[TEST test_expectation],
  "lexer" => %i[LEXER lexer_entries],
  "convert" => %i[CONVERT convert_name], "pragma" => %i[PRAGMA pragma_value],
  "display" => %i[DISPLAY display_symbol], "type" => %i[TYPE type_symbol],
  "param" => %i[PARAM param_name],
  "printer" => %i[PRINTER printer_symbol],
  "rule" => %i[RULE rules]
}.freeze
ASSOCIATIONS =

Signature:

  • Hash[String, [external_token, Symbol]]

Returns:

  • (Hash[String, external_token]) 
{
  "left" => :LEFT, "right" => :RIGHT, "nonassoc" => :NONASSOC, "precedence" => :PRECEDENCE
}.freeze
SCALAR_TYPES =

Signature:

  • Hash[String, external_token]

Returns:

  • (Hash[Symbol, external_token]) 
{
  literal: :LITERAL, regexp: :REGEXP, integer: :INTEGER, action: :ACTION, user_code: :USER_CODE
}.freeze
EXPECTATIONS =

Signature:

  • Hash[Symbol, external_token]

Returns:

  • (Hash[Symbol, String]) 
{
  class_keyword: "class", class_name: "identifier", superclass_name: "identifier",
  expect_integer: "integer", expect_rr_integer: "integer",
  start_first_symbol: "a grammar symbol", start_symbols: "a grammar symbol",
  include_path: "a double-quoted relative path",
  display_symbol: "a grammar symbol", type_symbol: "a grammar symbol",
  display_value: "a quoted string", type_value: "a quoted string",
  param_name: "an identifier", param_type: "a quoted type or declaration",
  printer_symbol: "a grammar symbol", printer_action: "an action",
  recover_kind: "sync", recovery_colon: ":", recovery_first_symbol: "a grammar symbol",
  recovery_symbols: "a grammar symbol", on_error_reduce_first_symbol: "a grammar symbol",
  on_error_reduce_symbols: "a grammar symbol", test_expectation: "accept or reject",
  test_source: "a double-quoted string",
  lexer_entries: "a lexer rule, state, or end", lexer_state_name: "a state name",
  lexer_state_do: "do", lexer_pattern: "a regular expression or quoted literal",
  lexer_action_or_entry: "an action, lexer rule, state, or end"
}.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from DeclarationLexerState

#begin_lexer_entry, #classify_lexer_identifier, #classify_lexer_scalar, #finish_lexer_scope, #lexer_declaration_state?

Methods included from DeclarationDocumentState

#class_keyword, #classify_include, #reject_fragment_pragma

Constructor Details

#initialize(extended: false) ⇒ DeclarationState

Returns a new instance of DeclarationState.

RBS:

  • (?extended: bool) -> void

Parameters:

  • extended: (Boolean) (defaults to: false) 


66
67
68
69
70
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 66

def initialize(extended: false)
  @extended_mode = extended
  @pragmas = {} #: Hash[String, bool]
  @state = :class_keyword
end

Instance Attribute Details

#conversion_name ⇒ Token? (readonly)

Signature:

  • Hash[Symbol, String]

Returns:



55
56
57
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 55

def conversion_name
  @conversion_name
end

#declaration ⇒ Symbol? (readonly)

Signature:

  • Symbol?

Returns:

  • (Symbol, nil) 


56
57
58
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 56

def declaration
  @declaration
end

#precedence_closer ⇒ String? (readonly)

Signature:

  • String?

Returns:

  • (String, nil) 


57
58
59
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 57

def precedence_closer
  @precedence_closer
end

#state ⇒ Symbol (readonly)

Signature:

  • Symbol

Returns:

  • (Symbol) 


58
59
60
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 58

def state
  @state
end

Instance Method Details

#begin_conversion(token, type, remaining) ⇒ external_token

RBS:

  • (Token token, external_token type, Array[Token] remaining) -> external_token

Parameters:

  • token (Token)
  • type (external_token)
  • remaining (Array[Token])

Returns:

  • (external_token) 


236
237
238
239
240
241
242
243
244
245
246
247
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 236

def begin_conversion(token, type, remaining)
  if token.type == :identifier && token.value == "end"
    @state = :declaration
    @declaration = nil
    return :END
  end

  validate_conversion_line(token, remaining)
  @conversion_name = token
  @state = :convert_expression
  type
end

#begin_declaration(token) ⇒ external_token

RBS:

  • (Token token) -> external_token

Parameters:

Returns:

  • (external_token) 


157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 157

def begin_declaration(token)
  value = string_value(token)
  return begin_precedence(token) if %w[prechigh preclow].include?(value)

  reject_fragment_pragma(token, value)

  terminal, next_state = DECLARATIONS[value]
  return :IDENTIFIER unless terminal

  @pragma_location = token.location if terminal == :PRAGMA
  @token_alias_candidate = nil
  @state = next_state
  @declaration = value.to_sym unless terminal == :RULE
  @declaration = nil if terminal == :RULE
  terminal
end

#begin_metadata_value(type) ⇒ external_token

RBS:

  • (external_token type) -> external_token

Parameters:

  • type (external_token)

Returns:

  • (external_token) 


352
353
354
355
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 352

def begin_metadata_value(type)
  @state = @state == :display_symbol ? :display_value : :type_value
  type
end

#begin_param_type ⇒ external_token

RBS:

  • () -> external_token

Returns:

  • (external_token) 


358
359
360
361
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 358

def begin_param_type
  @state = :param_type
  :IDENTIFIER
end

#begin_precedence(token) ⇒ external_token

RBS:

  • (Token token) -> external_token

Parameters:

Returns:

  • (external_token) 


190
191
192
193
194
195
196
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 190

def begin_precedence(token)
  high_to_low = string_value(token) == "prechigh"
  @precedence_closer = high_to_low ? "preclow" : "prechigh"
  @declaration = :precedence
  @state = :precedence_association
  high_to_low ? :PRECHIGH : :PRECLOW
end

#begin_printer_action(type) ⇒ external_token

RBS:

  • (external_token type) -> external_token

Parameters:

  • type (external_token)

Returns:

  • (external_token) 


364
365
366
367
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 364

def begin_printer_action(type)
  @state = :printer_action
  type
end

#begin_recovery_colon(_token) ⇒ external_token

RBS:

  • (Token token) -> external_token

Parameters:

Returns:

  • (external_token) 


370
371
372
373
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 370

def begin_recovery_colon(_token)
  @state = :recovery_colon
  :IDENTIFIER
end

#begin_test_source(_token) ⇒ external_token

RBS:

  • (Token token) -> external_token

Parameters:

Returns:

  • (external_token) 


376
377
378
379
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 376

def begin_test_source(_token)
  @state = :test_source
  :IDENTIFIER
end

#classify(token, remaining) ⇒ external_token

RBS:

  • (Token token, Array[Token] remaining) -> external_token

Parameters:

Returns:

  • (external_token) 


73
74
75
76
77
78
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 73

def classify(token, remaining)
  return classify_identifier(token, remaining) if token.type == :identifier
  return classify_scalar(token, remaining) if SCALAR_TYPES.key?(token.type)

  classify_punctuation(token)
end

#classify_conversion(token, type, remaining) ⇒ external_token?

RBS:

  • (Token token, external_token type, Array[Token] remaining) -> external_token?

Parameters:

  • token (Token)
  • type (external_token)
  • remaining (Array[Token])

Returns:

  • (external_token, nil) 


319
320
321
322
323
324
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 319

def classify_conversion(token, type, remaining)
  return unless type == :LITERAL
  return begin_conversion(token, type, remaining) if @state == :convert_name

  finish_conversion(type) if @state == :convert_expression
end

#classify_grammar_test(type) ⇒ external_token?

RBS:

  • (external_token type) -> external_token?

Parameters:

  • type (external_token)

Returns:

  • (external_token, nil) 


310
311
312
313
314
315
316
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 310

def classify_grammar_test(type)
  return unless @state == :test_source && type == :LITERAL

  @state = :declaration
  @declaration = nil
  type
end

#classify_identifier(token, remaining) ⇒ external_token

RBS:

  • (Token token, Array[Token] remaining) -> external_token

Parameters:

Returns:

  • (external_token) 


114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 114

def classify_identifier(token, remaining)
  return classify_lexer_identifier(token) if lexer_declaration_state?

  case @state
  when :class_keyword then class_keyword(token)
  when :class_name, :superclass_name then constant_name(remaining)
  when :declaration, :param_type then begin_declaration(token)
  when :token_symbols
    @token_alias_candidate = token
    declaration_symbol(token)
  when :options_identifiers, :start_symbols, :recovery_symbols, :on_error_reduce_symbols
    declaration_symbol(token)
  when :precedence_association, :precedence_symbols then precedence_identifier(token)
  when :start_first_symbol then continue_start_symbols(:IDENTIFIER)
  when :display_symbol, :type_symbol then begin_metadata_value(:IDENTIFIER)
  when :param_name then begin_param_type
  when :printer_symbol then begin_printer_action(:IDENTIFIER)
  when :recover_kind then begin_recovery_colon(token)
  when :recovery_first_symbol then continue_recovery_symbols(:IDENTIFIER)
  when :on_error_reduce_first_symbol then continue_on_error_reduce_symbols(:IDENTIFIER)
  when :test_expectation then begin_test_source(token)
  when :pragma_value then finish_pragma(token)
  when :convert_name then begin_conversion(token, :IDENTIFIER, remaining)
  else :IDENTIFIER
  end
end

#classify_metadata(type) ⇒ external_token?

RBS:

  • (external_token type) -> external_token?

Parameters:

  • type (external_token)

Returns:

  • (external_token, nil) 


291
292
293
294
295
296
297
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 291

def classify_metadata(type)
  return unless type == :LITERAL
  return begin_metadata_value(type) if %i[display_symbol type_symbol].include?(@state)
  return finish_param_type(type) if @state == :param_type

  finish_metadata_value(type) if %i[display_value type_value].include?(@state)
end

#classify_printer(type) ⇒ external_token?

RBS:

  • (external_token type) -> external_token?

Parameters:

  • type (external_token)

Returns:

  • (external_token, nil) 


300
301
302
303
304
305
306
307
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 300

def classify_printer(type)
  return begin_printer_action(type) if @state == :printer_symbol && type == :LITERAL
  return unless @state == :printer_action && type == :ACTION

  @state = :declaration
  @declaration = nil
  type
end

#classify_punctuation(token) ⇒ external_token

RBS:

  • (Token token) -> external_token

Parameters:

Returns:

  • (external_token) 


403
404
405
406
407
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 403

def classify_punctuation(token)
  @state = :superclass_name if @state == :superclass_marker && token.type == :<
  @state = :recovery_first_symbol if @state == :recovery_colon && token.type == :":"
  string_value(token)
end

#classify_scalar(token, remaining) ⇒ external_token

RBS:

  • (Token token, Array[Token] remaining) -> external_token

Parameters:

Returns:

  • (external_token) 


250
251
252
253
254
255
256
257
258
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 250

def classify_scalar(token, remaining)
  type = SCALAR_TYPES.fetch(token.type)
  classified = classify_lexer_scalar(type) ||
               classify_token_alias(token, type) || classify_include(type) || classify_single_symbol(type) ||
               classify_metadata(type) || classify_printer(type) || classify_grammar_test(type) ||
               classify_conversion(token, type, remaining)

  classified || type
end

#classify_single_symbol(type) ⇒ external_token?

RBS:

  • (external_token type) -> external_token?

Parameters:

  • type (external_token)

Returns:

  • (external_token, nil) 


261
262
263
264
265
266
267
268
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 261

def classify_single_symbol(type)
  return finish_single_symbol(type) if %i[expect_integer expect_rr_integer].include?(@state) && type == :INTEGER

  first = continue_first_symbol(type)
  return first if first

  type if @state == :start_symbols && type == :LITERAL
end

#classify_token_alias(token, type) ⇒ external_token?

RBS:

  • (Token token, external_token type) -> external_token?

Parameters:

  • token (Token)
  • type (external_token)

Returns:

  • (external_token, nil) 


280
281
282
283
284
285
286
287
288
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 280

def classify_token_alias(token, type)
  candidate = @token_alias_candidate
  return unless extended_features? && @state == :token_symbols &&
                type == :LITERAL && candidate
  return unless candidate.location.line == token.location.line

  @token_alias_candidate = nil
  :TOKEN_ALIAS
end

#constant_name(remaining) ⇒ external_token

RBS:

  • (Array[Token] remaining) -> external_token

Parameters:

  • remaining (Array[Token])

Returns:

  • (external_token) 


142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 142

def constant_name(remaining)
  following = remaining.first
  raise Ibex::Error, "unexpected end of token stream" unless following

  @state = if following.type == :scope
             @state
           elsif following.type == :<
             :superclass_marker
           else
             :declaration
           end
  :IDENTIFIER
end

#continue_first_symbol(type) ⇒ external_token?

RBS:

  • (external_token type) -> external_token?

Parameters:

  • type (external_token)

Returns:

  • (external_token, nil) 


271
272
273
274
275
276
277
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 271

def continue_first_symbol(type)
  return unless type == :LITERAL
  return continue_start_symbols(type) if @state == :start_first_symbol
  return continue_recovery_symbols(type) if @state == :recovery_first_symbol

  continue_on_error_reduce_symbols(type) if @state == :on_error_reduce_first_symbol
end

#continue_on_error_reduce_symbols(type) ⇒ external_token

RBS:

  • (external_token type) -> external_token

Parameters:

  • type (external_token)

Returns:

  • (external_token) 


346
347
348
349
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 346

def continue_on_error_reduce_symbols(type)
  @state = :on_error_reduce_symbols
  type
end

#continue_recovery_symbols(type) ⇒ external_token

RBS:

  • (external_token type) -> external_token

Parameters:

  • type (external_token)

Returns:

  • (external_token) 


340
341
342
343
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 340

def continue_recovery_symbols(type)
  @state = :recovery_symbols
  type
end

#continue_start_symbols(type) ⇒ external_token

RBS:

  • (external_token type) -> external_token

Parameters:

  • type (external_token)

Returns:

  • (external_token) 


334
335
336
337
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 334

def continue_start_symbols(type)
  @state = :start_symbols
  type
end

#cst_pragma? ⇒ Boolean

RBS:

  • () -> bool

Returns:

  • (Boolean) 


91
92
93
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 91

def cst_pragma?
  @pragmas["cst"] == true
end

#declaration_boundary?(value) ⇒ Boolean

RBS:

  • (String value) -> bool

Parameters:

  • value (String)

Returns:

  • (Boolean) 


206
207
208
209
210
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 206

def declaration_boundary?(value)
  return false if %w[display type param printer].include?(value) && !extended_features?

  DECLARATIONS.key?(value) || %w[prechigh preclow].include?(value)
end

#declaration_symbol(token) ⇒ external_token

RBS:

  • (Token token) -> external_token

Parameters:

Returns:

  • (external_token) 


199
200
201
202
203
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 199

def declaration_symbol(token)
  return begin_declaration(token) if declaration_boundary?(string_value(token))

  :IDENTIFIER
end

#expectation(token) ⇒ String?

RBS:

  • (Token? token) -> String?

Parameters:

Returns:

  • (String, nil) 


96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 96

def expectation(token)
  return "identifier" if @state == :pragma_value && token&.type != :identifier

  expected = EXPECTATIONS[@state]
  return expected if expected

  if @declaration == :precedence
    precedence_expectation(token)
  elsif @declaration == :convert
    "end"
  elsif @state == :declaration
    token&.type == :eof ? "rule" : "a declaration or rule"
  end
end

#extended_features? ⇒ Boolean

RBS:

  • () -> bool

Returns:

  • (Boolean) 


213
214
215
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 213

def extended_features?
  @extended_mode || extended_pragma? || cst_pragma?
end

#extended_pragma? ⇒ Boolean

RBS:

  • () -> bool

Returns:

  • (Boolean) 


86
87
88
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 86

def extended_pragma?
  @pragmas["extended"] == true
end

#finish_conversion(type) ⇒ external_token

RBS:

  • (external_token type) -> external_token

Parameters:

  • type (external_token)

Returns:

  • (external_token) 


396
397
398
399
400
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 396

def finish_conversion(type)
  @state = :convert_name
  @conversion_name = nil
  type
end

#finish_metadata_value(type) ⇒ external_token

RBS:

  • (external_token type) -> external_token

Parameters:

  • type (external_token)

Returns:

  • (external_token) 


389
390
391
392
393
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 389

def finish_metadata_value(type)
  @state = :declaration
  @declaration = nil
  type
end

#finish_param_type(type) ⇒ external_token

RBS:

  • (external_token type) -> external_token

Parameters:

  • type (external_token)

Returns:

  • (external_token) 


382
383
384
385
386
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 382

def finish_param_type(type)
  @state = :declaration
  @declaration = nil
  type
end

#finish_pragma(token) ⇒ external_token

RBS:

  • (Token token) -> external_token

Parameters:

Returns:

  • (external_token) 


175
176
177
178
179
180
181
182
183
184
185
186
187
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 175

def finish_pragma(token)
  value = string_value(token)
  raise Ibex::Error, "#{token.location}: unknown pragma #{value}" unless %w[extended cst].include?(value)

  location = @pragma_location || token.location
  raise Ibex::Error, "#{location}: duplicate pragma #{value}" if @pragmas[value]

  @pragmas[value] = true
  @pragma_location = nil
  @state = :declaration
  @declaration = nil
  :IDENTIFIER
end

#finish_precedence(token) ⇒ external_token

RBS:

  • (Token token) -> external_token

Parameters:

Returns:

  • (external_token) 


228
229
230
231
232
233
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 228

def finish_precedence(token)
  @state = :declaration
  @declaration = nil
  @precedence_closer = nil
  string_value(token) == "prechigh" ? :PRECHIGH : :PRECLOW
end

#finish_single_symbol(type) ⇒ external_token

RBS:

  • (external_token type) -> external_token

Parameters:

  • type (external_token)

Returns:

  • (external_token) 


327
328
329
330
331
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 327

def finish_single_symbol(type)
  @state = :declaration
  @declaration = nil
  type
end

#precedence_expectation(token) ⇒ String?

RBS:

  • (Token? token) -> String?

Parameters:

Returns:

  • (String, nil) 


421
422
423
424
425
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 421

def precedence_expectation(token)
  return @precedence_closer if token&.type == :eof

  "left or right or nonassoc or precedence" if @state == :precedence_association
end

#precedence_identifier(token) ⇒ external_token

RBS:

  • (Token token) -> external_token

Parameters:

Returns:

  • (external_token) 


218
219
220
221
222
223
224
225
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 218

def precedence_identifier(token)
  value = string_value(token)
  return finish_precedence(token) if value == @precedence_closer

  association = ASSOCIATIONS[value]
  @state = :precedence_symbols if association
  association || :IDENTIFIER
end

#rules? ⇒ Boolean

RBS:

  • () -> bool

Returns:

  • (Boolean) 


81
82
83
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 81

def rules?
  @state == :rules
end

#string_value(token) ⇒ String

RBS:

  • (Token token) -> String

Parameters:

Returns:

  • (String) 


428
429
430
431
432
433
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 428

def string_value(token)
  value = token.value
  return value if value.is_a?(String)

  raise Ibex::Error, "#{token.location}: expected text token"
end

#validate_conversion_line(name, remaining) ⇒ void

This method returns an undefined value.

RBS:

  • (Token name, Array[Token] remaining) -> void

Parameters:



410
411
412
413
414
415
416
417
418
 
# File 'lib/ibex/frontend/token_adapter/declaration_state.rb', line 410

def validate_conversion_line(name, remaining)
  line = name.location.line
  rest = remaining.take_while do |token|
    token.type != :eof && token.location.line == line && !(token.type == :identifier && token.value == "end")
  end
  return if rest.length == 1 && rest.first.type == :literal

  raise Ibex::Error, "#{name.location}: expected a quoted Ruby conversion expression"
end