Class: Luoma::UnifiedLexer

Inherits:
BaseLexer show all
Defined in:
lib/luoma/lexer_unified.rb,
sig/luoma/lexer_unified.rbs

Overview

A single-pass template tokenizer for the Unified Expression Language with new style comments and no line statements.

https://jg-rp.github.io/template-expression-spec/

Constant Summary collapse

RE_FLOAT =

Returns:

  • (::Regexp) 
/((?:\d+\.\d+(?:[eE][+-]?\d+)?)|(\d+[eE]-\d+))/
RE_INT =

Returns:

  • (::Regexp) 
/\d+(?:[eE]\+?\d+)?/
RE_MARKUP_START =

Returns:

  • (::Regexp) 
/\{[%{#]/
RE_OUTPUT_END =

Returns:

  • (::Regexp) 
/\}\}/
RE_PUNCTUATION =

Returns:

  • (::Regexp) 
/!=|>=|<=|==|=>|->|\.{1,3}|[?\[\]|:,()*\/<>=]|([+-](?![}%#]\}))|(%(?!\}))/
RE_RAW_END =

Returns:

  • (::Regexp) 
/\{%[+~-]?\s*endraw\s*[+~-]?%\}/
RE_TAG_END =

Returns:

  • (::Regexp) 
/%\}/
RE_TAG_NAME =

Returns:

  • (::Regexp) 
/[a-z][a-z_0-9]*/
RE_TRIVIA =

Returns:

  • (::Regexp) 
/[ \n\r\t]+/
RE_WHITESPACE_CONTROL =

Returns:

  • (::Regexp) 
/[+\-~]/
RE_WORD =

Returns:

  • (::Regexp) 
/[\u0080-\uFFFFa-zA-Z_][\u0080-\uFFFFa-zA-Z0-9_-]*/
RE_HASH_COUNT =

Returns:

  • (Hash[Integer, Regexp]) 
{
  1 => /[+\-~]?\#\}/,
  2 => /[+\-~]?\#{2}\}/,
  3 => /[+\-~]?\#{3}\}/,
  4 => /[+\-~]?\#{4}\}/,
  5 => /[+\-~]?\#{5}\}/,
  6 => /[+\-~]?\#{6}\}/,
  7 => /[+\-~]?\#{7}\}/
}.freeze
TOKEN_MAP =

Keywords and symbols that get their own token kind.

Returns:

  • (Hash[String, t_token_kind]) 
{
  "true" => :token_true,
  "false" => :token_false,
  "nil" => :token_nil,
  "null" => :token_nil,
  "and" => :token_and,
  "or" => :token_or,
  "orElse" => :token_or_else,
  "not" => :token_not,
  "contains" => :token_contains,
  "in" => :token_in,
  "if" => :token_if,
  "else" => :token_else,
  "?" => :token_question,
  "[" => :token_lbracket,
  "]" => :token_rbracket,
  "|" => :token_pipe,
  "." => :token_dot,
  ".." => :token_double_dot,
  "..." => :token_triple_dot,
  "," => :token_comma,
  ":" => :token_colon,
  "(" => :token_lparen,
  ")" => :token_rparen,
  "=" => :token_assign,
  "<" => :token_lt,
  "<=" => :token_le,
  "<>" => :token_ne,
  ">" => :token_gt,
  ">=" => :token_ge,
  "==" => :token_eq,
  "!=" => :token_ne,
  "=>" => :token_arrow,
  "->" => :token_arrow,
  "+" => :token_add,
  "-" => :token_sub,
  "%" => :token_mod,
  "*" => :token_mul,
  "/" => :token_div
}.freeze

Instance Attribute Summary

Attributes inherited from BaseLexer

#tokens

Instance Method Summary collapse

Methods inherited from BaseLexer

#emit, #go, #index, #initialize, #scan, #scan_until?, #skip?, #skip_until?, tokenize

Constructor Details

This class inherits a constructor from Luoma::BaseLexer

Instance Method Details

#accept_comment ⇒ void

This method returns an undefined value.

() -> void



166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
 
# File 'lib/luoma/lexer_unified.rb', line 166

def accept_comment
  start_of_delim = @start
  @scanner.pos += 1 while @scanner.peek_byte == 35 # steep:ignore
  hash_count = (@source.byteslice((@start + 1)...@scanner.pos) || raise).size
  re = RE_HASH_COUNT[hash_count]

  emit(:token_comment_start)
  wc = accept_whitespace_control?

  if re && scan_until?(re)
    emit(:token_comment)
    accept_whitespace_control?
    @scanner.scan(re)
    emit(:token_comment_end)
  else
    # No closing delimiter. Not a comment.
    @tokens.pop
    @tokens.pop if wc
    @start = start_of_delim
    emit(:token_text)
  end
end

#accept_expression ⇒ void

This method returns an undefined value.

() -> void



134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'lib/luoma/lexer_unified.rb', line 134

def accept_expression
  loop do
    skip?(RE_TRIVIA)

    if (match = @scanner.scan(RE_PUNCTUATION))
      emit(TOKEN_MAP[match] || :token_unknown)
    elsif (match = @scanner.scan(RE_WORD))
      emit(TOKEN_MAP[match] || :token_ident)
    elsif @scanner.scan(RE_FLOAT)
      emit(:token_float)
    elsif @scanner.scan(RE_INT)
      emit(:token_int)
    else
      case @scanner.peek_byte # steep:ignore
      when 39, 34 # ' or "
        # String literals get their own state because we allow markup
        # delimiters inside quotes.
        accept_string
      when 123 # {
        # Object literals require their own state so we can tell the
        # difference between a closing brace and the start of a closing
        # output delimiter.
        accept_object
      else
        # Non-expression byte or end of input.
        break
      end
    end
  end
end

#accept_object ⇒ void

This method returns an undefined value.

() -> void



250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/luoma/lexer_unified.rb', line 250

def accept_object
  @scanner.pos += 1
  emit(:token_lbrace)

  loop do
    case @scanner.peek_byte # steep:ignore
    when 125 # }
      @scanner.pos += 1
      emit(:token_rbrace)
      break
    when nil
      # Unclosed object literal.
      break
    else
      accept_expression
    end
  end
end

#accept_string ⇒ void

This method returns an undefined value.

() -> void



190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
 
# File 'lib/luoma/lexer_unified.rb', line 190

def accept_string
  # `get_byte` returns a string, `peek_byte` returns an integer :(
  quote = @scanner.get_byte || raise
  byte = quote.ord
  double = quote == '"'

  quote_kind = double ? :token_double_quote : :token_single_quote #: t_token_kind
  unescaped_kind = double ? :token_double_quoted : :token_single_quoted #: t_token_kind
  escaped_kind = double ? :token_double_escaped : :token_single_escaped #: t_token_kind
  current_kind = unescaped_kind #: t_token_kind

  emit(quote_kind)

  if @scanner.peek_byte == byte # steep:ignore
    @scanner.pos += 1
    emit(quote_kind)
    return
  end

  loop do
    case @scanner.get_byte
    when quote
      @scanner.pos -= 1
      emit(current_kind) if @start < @scanner.pos
      @scanner.pos += 1
      emit(quote_kind)
      break
    when "\\"
      @scanner.pos -= 1
      # Emit unescaped segment, if any.
      emit(current_kind) if current_kind == unescaped_kind && @start < @scanner.pos
      @scanner.pos += 2
      current_kind = escaped_kind
    when "$"
      next unless @scanner.peek_byte == 123 # steep:ignore

      @scanner.pos -= 1
      emit(current_kind) if @start < @scanner.pos

      current_kind = unescaped_kind
      @scanner.pos += 2
      emit(:token_interpolation_start)

      accept_expression

      if @scanner.peek_byte == 125 # steep:ignore
        @scanner.pos += 1
        emit(:token_interpolation_end)
      else
        # unclosed interpolation. Let the parser handle it.
        break
      end
    when nil
      # Unclosed string literal. Let the parser handle it.
      break
    end
  end
end

#accept_tag ⇒ void

This method returns an undefined value.

() -> void?



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/luoma/lexer_unified.rb', line 107

def accept_tag
  tag_name = @scanner.scan(RE_TAG_NAME)

  case tag_name
  when "raw"
    emit(:token_tag_name)
    skip?(RE_TRIVIA)
    accept_whitespace_control?
    emit(:token_tag_end) if @scanner.scan(RE_TAG_END)
    scan_until?(RE_RAW_END)
    emit(:token_text) if @start < @scanner.pos
  when nil
    # Missing or malformed tag name
    accept_expression
    skip?(RE_TRIVIA)
    accept_whitespace_control?
    emit(:token_tag_end) if @scanner.scan(RE_TAG_END)
  else
    emit(:token_tag_name)
    accept_expression
    skip?(RE_TRIVIA)
    accept_whitespace_control?
    emit(:token_tag_end) if @scanner.scan(RE_TAG_END)
  end
end

#accept_whitespace_control? ⇒ Boolean

() -> bool

Returns:

  • (Boolean) 


270
271
272
273
274
275
276
277
 
# File 'lib/luoma/lexer_unified.rb', line 270

def accept_whitespace_control?
  if @scanner.scan(RE_WHITESPACE_CONTROL)
    emit(:token_wc)
    true
  else
    false
  end
end

#scan_markup ⇒ Object

Signature:

  • () -> Symbol?



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/luoma/lexer_unified.rb', line 76

def scan_markup
  loop do
    case @scanner.scan(RE_MARKUP_START)
    when "{{"
      emit(:token_out_start)
      accept_whitespace_control?
      accept_expression
      skip?(RE_TRIVIA)
      accept_whitespace_control?
      emit(:token_out_end) if @scanner.scan(RE_OUTPUT_END)
    when "{%"
      emit(:token_tag_start)
      accept_whitespace_control?
      skip?(RE_TRIVIA)
      accept_tag
    when "{#"
      accept_comment
    else
      if scan_until?(RE_MARKUP_START)
        emit(:token_text)
      else
        # No more markup. Emit text to end of string.
        @scanner.terminate
        emit(:token_text) if @start < @scanner.pos
        return nil
      end
    end
  end
end