Module: Ibex::NormalizeLexer

Included in:
Normalizer
Defined in:
lib/ibex/normalize/lexer.rb,
sig/ibex/normalize/lexer.rbs

Overview

Lexer declaration normalization and static safety checks.

Instance Method Summary collapse

Instance Method Details

#decode_lexer_literal(raw, location) ⇒ String

RBS:

  • (String raw, Frontend::Location location) -> String

Parameters:

Returns:

  • (String) 


104
105
106
107
108
109
110
111
 
# File 'lib/ibex/normalize/lexer.rb', line 104

def decode_lexer_literal(raw, location)
  # @type self: Normalizer
  return raw.undump if raw.start_with?('"')

  (raw[1...-1] || "").gsub("\\'", "'").gsub("\\\\", "\\")
rescue RuntimeError => e
  fail_at(location, "invalid lexer literal: #{e.message}")
end

#normalize_lexer ⇒ IR::Lexer?

RBS:

  • () -> IR::Lexer?

Returns:



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/ibex/normalize/lexer.rb', line 9

def normalize_lexer
  # @type self: Normalizer
  declaration = @lexer_declaration
  return unless declaration

  states = ["INITIAL"] #: Array[String]
  rules = [] #: Array[IR::LexerRule]
  warnings = [] #: Array[Hash[Symbol, untyped]]
  declaration.definitions.each do |definition|
    if definition.is_a?(Frontend::AST::LexerState)
      normalize_lexer_state(definition, states, rules, warnings)
    else
      rules << normalize_lexer_rule(definition, "INITIAL", rules.length, warnings)
    end
  end
  fail_at(declaration.loc, "lexer declaration requires at least one rule") if rules.empty?
  IR::Lexer.new(
    states: states, rules: rules, warnings: warnings,
    source_provenance: { file: declaration.loc.file, root: @resolution&.root_directory, byte_span: nil }
  )
end

#normalize_lexer_pattern(definition) ⇒ [ String, String ]

RBS:

  • (Frontend::AST::LexerRule definition) -> [String, String]

Parameters:

Returns:

  • ([ String, String ]) 


89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/ibex/normalize/lexer.rb', line 89

def normalize_lexer_pattern(definition)
  # @type self: Normalizer
  raw = definition.pattern
  if definition.pattern_kind == :literal
    decoded = decode_lexer_literal(raw, definition.loc)
    fail_at(definition.loc, "lexer pattern must not be empty") if decoded.empty?
    return [Regexp.escape(decoded), ""]
  end

  closing = raw.rindex("/")
  fail_at(definition.loc, "invalid lexer regular expression") unless closing&.positive?
  [raw[1...closing] || "", raw[(closing + 1)..] || ""]
end

#normalize_lexer_rule(definition, state, id, warnings) ⇒ IR::LexerRule

RBS:

  • (Frontend::AST::LexerRule definition, String state, Integer id, Array[Hash[Symbol, untyped]] warnings) -> IR::LexerRule

Parameters:

Returns:



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/ibex/normalize/lexer.rb', line 47

def normalize_lexer_rule(definition, state, id, warnings)
  # @type self: Normalizer
  validate_lexer_token(definition)
  validate_lexer_action(definition)
  source, options = normalize_lexer_pattern(definition)
  validate_lexer_regexp(definition, source, options)
  if risky_lexer_pattern?(source)
    warning = { type: :lexer_redos, loc: definition.loc.to_h } #: IR::grammar_warning
    warning[:symbol] = definition.token if definition.token
    @warnings << warning
    warnings << { type: :redos, rule: id, loc: definition.loc.to_h }
  end
  IR::LexerRule.new(
    id: id, state: state, kind: definition.kind, token: definition.token,
    pattern: source, pattern_kind: definition.pattern_kind, options: options,
    action: definition.action, location: definition.loc.to_h
  )
end

#normalize_lexer_state(definition, states, rules, warnings) ⇒ void

This method returns an undefined value.

RBS:

  • (Frontend::AST::LexerState definition, Array[String] states, Array[IR::LexerRule] rules, Array[Hash[Symbol, untyped]] warnings) -> void

Parameters:



33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/ibex/normalize/lexer.rb', line 33

def normalize_lexer_state(definition, states, rules, warnings)
  # @type self: Normalizer
  fail_at(definition.loc, "lexer state INITIAL is reserved") if definition.name == "INITIAL"
  fail_at(definition.loc, "duplicate lexer state #{definition.name}") if states.include?(definition.name)
  states << definition.name
  definition.definitions.each do |entry|
    fail_at(entry.loc, "nested lexer states are not supported") if entry.is_a?(Frontend::AST::LexerState)

    rules << normalize_lexer_rule(entry, definition.name, rules.length, warnings)
  end
end

#risky_lexer_pattern?(source) ⇒ Boolean

RBS:

  • (String source) -> bool

Parameters:

  • source (String)

Returns:

  • (Boolean) 


127
128
129
 
# File 'lib/ibex/normalize/lexer.rb', line 127

def risky_lexer_pattern?(source)
  source.match?(/\([^)]*[+*][^)]*\)[+*{]/) || source.match?(/\.\*[+*{]/)
end

#validate_lexer_action(definition) ⇒ void

This method returns an undefined value.

RBS:

  • (Frontend::AST::LexerRule definition) -> void

Parameters:



76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/ibex/normalize/lexer.rb', line 76

def validate_lexer_action(definition)
  # @type self: Normalizer
  action = definition.action
  return unless action&.lstrip&.start_with?("|")

  match = action.match(/\A\s*\|([a-z_][a-zA-Z0-9_]*)\|/m)
  fail_at(definition.loc, "lexer action accepts exactly one local identifier") unless match
  name = match[1]
  fail_at(definition.loc, "lexer action parameter #{name.inspect} is a Ruby keyword") if
    Normalizer::RUBY_KEYWORDS.include?(name)
end

#validate_lexer_regexp(definition, source, options) ⇒ void

This method returns an undefined value.

RBS:

  • (Frontend::AST::LexerRule definition, String source, String options) -> void

Parameters:



114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/ibex/normalize/lexer.rb', line 114

def validate_lexer_regexp(definition, source, options)
  # @type self: Normalizer
  flags = 0
  flags |= Regexp::IGNORECASE if options.include?("i")
  flags |= Regexp::MULTILINE if options.include?("m")
  flags |= Regexp::EXTENDED if options.include?("x")
  regexp = Regexp.new("\\A(?:#{source})", flags)
  fail_at(definition.loc, "lexer pattern must not match an empty string") if regexp.match?("")
rescue RegexpError => e
  fail_at(definition.loc, "invalid lexer regular expression: #{e.message}")
end

#validate_lexer_token(definition) ⇒ void

This method returns an undefined value.

RBS:

  • (Frontend::AST::LexerRule definition) -> void

Parameters:



67
68
69
70
71
72
73
 
# File 'lib/ibex/normalize/lexer.rb', line 67

def validate_lexer_token(definition)
  # @type self: Normalizer
  return unless definition.kind == :token
  return if definition.token && @declared_tokens.key?(definition.token)

  fail_at(definition.loc, "lexer rule references undeclared terminal #{definition.token}")
end