Class: Luoma::LegacyLexer

Inherits:
BaseLexer show all
Defined in:
lib/luoma/lexer_legacy.rb,
sig/luoma/lexer_legacy.rbs

Overview

A single-pass template tokenizer that matches Shopify/liquid v5.12.0 strict mode syntax and semantics.

We use lookahead and byte offset limits to achieve behavior equivalent to Shopify's two-pass tokenizer/parser.

Constant Summary collapse

RE_MARKUP =

Returns:

  • (::Regexp) 
/\{[%{]/
RE_OUT_END =

Returns:

  • (::Regexp) 
/-?(\}\}?|%\}(?!\}))/
RE_TAG_END =

Returns:

  • (::Regexp) 
/-?%\}/
RE_TRIVIA =

Returns:

  • (::Regexp) 
/[ \n\r\t\f]+/
RE_LINE_TRIVIA =

Returns:

  • (::Regexp) 
/[ \t\f\r]+/
RE_TAG_NAME =

Returns:

  • (::Regexp) 
/#|[a-zA-Z0-9_]+/
RE_PUNCTUATION =

Returns:

  • (::Regexp) 
/[.!=<>]{1,2}|[?\[\]|:,()]/
RE_IDENT =

Returns:

  • (::Regexp) 
/[a-zA-Z_][a-zA-Z0-9_-]*\??/
RE_FLOAT =

Returns:

  • (::Regexp) 
/-?\d+\.\d+/
RE_INT =

Returns:

  • (::Regexp) 
/-?\d+/
RE_COMMENT_SEGMENT =

Returns:

  • (::Regexp) 
/\{%-?\s*(comment|raw|endcomment|endraw).*?-?%\}/m
RE_LINE_COMMENT_SEGMENT =

Returns:

  • (::Regexp) 
/\n\s*(comment|endcomment).*/
RE_END_DOC =

Returns:

  • (::Regexp) 
/\{%-?\s*enddoc\s*-?%\}/
RE_END_RAW =

Returns:

  • (::Regexp) 
/\{%-?\s*endraw\s*-?%\}/
TOKEN_MAP =

Keywords and symbols that get their own token kind.

Returns:

  • (Hash[String, t_token_kind]) 
{
  "true" => :token_true,
  "false" => :token_false,
  "nil" => :token_nil,
  "null" => :token_nil,
  "and" => :token_and,
  "or" => :token_or,
  "contains" => :token_contains,
  "in" => :token_in,
  "blank" => :token_blank,
  "empty" => :token_empty,
  "[" => :token_lbracket,
  "]" => :token_rbracket,
  "|" => :token_pipe,
  "." => :token_dot,
  ".." => :token_double_dot,
  "," => :token_comma,
  ":" => :token_colon,
  "(" => :token_lparen,
  ")" => :token_rparen,
  "=" => :token_assign,
  "<" => :token_lt,
  "<=" => :token_le,
  "<>" => :token_ne,
  ">" => :token_gt,
  ">=" => :token_ge,
  "==" => :token_eq,
  "!=" => :token_ne,
  "#" => :token_hash
}.freeze
RE_UP_TO_OUT_END =

Returns:

  • (::Regexp) 

      
        
RE_UP_TO_TAG_END =
          

  

    

  





  

Returns:


    
  
    
  • 
      
      
        (::Regexp)
      
      
      
    
    • 
  





        

        

      
    

  






  
Instance Attribute Summary

  
  
Attributes inherited from BaseLexer

  
#tokens



  
    

      Instance Method Summary
      collapse
    


    
    
  


  
  
  
  
  
  
  
  
  
  
Methods inherited from BaseLexer

  
#emit, #go, #index, #initialize, #scan, #scan_until?, #skip?, #skip_until?, tokenize



  
Constructor Details

  
    
This class inherits a constructor from Luoma::BaseLexer

  




  

    
Instance Method Details


    
      

  

  
    #accept_block_comment(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 199

def accept_block_comment(limit)
  skip?(RE_TRIVIA)
  # Ignore any "expression".
  @scanner.pos = limit
  accept_whitespace_control?
  @scanner.scan(RE_TAG_END)
  emit(:token_tag_end)

  comment_depth = 1
  raw_depth = 0

  loop do
    match = skip_until?(RE_COMMENT_SEGMENT)

    unless match
      emit(:token_unknown)
      break
    end

    case @scanner[1]
    when "comment"
      comment_depth += 1
      @scanner.pos += @scanner.matched_size || raise
    when "raw"
      raw_depth += 1
      @scanner.pos += @scanner.matched_size || raise
    when "endraw"
      raw_depth -= 1 if raw_depth.positive?
      @scanner.pos += @scanner.matched_size || raise
    when "endcomment"
      if raw_depth.positive?
        @scanner.pos += @scanner.matched_size || raise
        next
      end

      comment_depth -= 1

      if comment_depth.positive?
        @scanner.pos += @scanner.matched_size || raise
        next
      end

      emit(:token_comment)
      break
    else
      raise "unreachable"
    end
  end
end

    		
  





    
      

  

  
    #accept_doc_comment(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


250
251
252
253
254
255
256
257
258

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 250

def accept_doc_comment(limit)
  # Let the parser handle unexpected expression tokens.
  accept_expression(limit)
  skip?(RE_TRIVIA)
  accept_whitespace_control?
  @scanner.scan(RE_TAG_END)
  emit(:token_tag_end)
  emit(:token_comment) if scan_until?(RE_END_DOC)
end

    		
  





    
      

  

  
    #accept_expression(limit, trivia = RE_TRIVIA)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  
    
  • 
      
        trivia
      
      
        (Regexp)
      
      
        (defaults to: RE_TRIVIA)
      
      
    
    • 
  






  

    
      


146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 146

def accept_expression(limit, trivia = RE_TRIVIA)
  loop do
    skip?(trivia)

    # Trivia can put us past the limit
    break if @scanner.pos >= limit

    # We assume punctuation does not include markup delimiter characters,
    # and can therefore not exceed `limit`.
    if (match = @scanner.scan(RE_PUNCTUATION))
      emit(TOKEN_MAP[match] || :token_unknown)
      next
    end

    # We assume identifiers do not allow markup delimiter characters,
    # and can therefore not exceed `limit`.
    if (match = @scanner.scan(RE_IDENT))
      emit(TOKEN_MAP[match] || :token_ident)
      next
    end

    # RE_FLOAT must come before RE_INT
    if @scanner.scan(RE_FLOAT)
      emit(:token_float)
      next
    end

    if @scanner.scan(RE_INT)
      emit(:token_int)
      next
    end

    next_byte = @scanner.peek_byte # steep:ignore

    if next_byte == 39 || next_byte == 34
      accept_string_literal(limit)
    else
      @scanner.pos += 1
      emit(:token_unknown)
    end
  end
end

    		
  





    
      

  

  
    #accept_inline_comment(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


190
191
192
193
194
195
196

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 190

def accept_inline_comment(limit)
  @scanner.pos = limit
  emit(:token_comment)
  accept_whitespace_control?
  @scanner.scan(RE_TAG_END)
  emit(:token_tag_end)
end

    		
  





    
      

  

  
    #accept_line_block_comment(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 316

def accept_line_block_comment(limit)
  comment_depth = 1

  while @scanner.pos < limit
    unless skip_until?(RE_LINE_COMMENT_SEGMENT)
      emit(:token_unknown)
      break
    end

    case @scanner[1]
    when "comment"
      comment_depth += 1
      @scanner.pos += @scanner.matched_size || raise
    when "endcomment"
      comment_depth -= 1
      if comment_depth.positive?
        @scanner.pos += @scanner.matched_size || raise
        next
      else
        emit(:token_comment)
        break
      end
    else
      raise "unreachable"
    end
  end
end

    		
  





    
      

  

  
    #accept_line_doc_comment(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


345
346
347
348
349

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 345

def accept_line_doc_comment(limit)
  # Shopify/liquid always raises a syntax error for `doc` in `{% liquid %}`.
  @scanner.pos = limit
  emit(:token_unknown)
end

    		
  





    
      

  

  
    #accept_line_raw_tag(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


352
353
354
355
356

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 352

def accept_line_raw_tag(limit)
  # Shopify/liquid always raises a syntax error for `raw` in `{% liquid %}`.
  @scanner.pos = limit
  emit(:token_unknown)
end

    		
  





    
      

  

  
    #accept_line_statements(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 272

def accept_line_statements(limit)
  while @scanner.pos < limit
    skip?(RE_TRIVIA)
    line_limit = @scanner.string.byteindex("\n", @scanner.pos) || limit
    line_limit = limit if line_limit > limit

    emit(:token_tag_start)

    case @scanner.scan(RE_TAG_NAME)
    when "#"
      emit(:token_tag_name)
      @scanner.pos = line_limit
      emit(:token_comment)
      emit(:token_tag_end)
    when "comment"
      emit(:token_tag_name)
      emit(:token_tag_end)
      accept_line_block_comment(limit)
    when "doc"
      emit(:token_tag_name)
      accept_line_doc_comment(limit)
    when "raw"
      emit(:token_tag_name)
      accept_line_raw_tag(limit)
    when "liquid"
      emit(:token_tag_name)
      accept_line_statements(line_limit)
    when nil
      # Remove empty :token_tag_start
      @tokens.pop
    else
      emit(:token_tag_name)
      accept_expression(line_limit, RE_LINE_TRIVIA)
      emit(:token_tag_end)
    end
  end

  skip?(RE_TRIVIA)
  accept_whitespace_control?
  @scanner.scan(RE_TAG_END)
  emit(:token_tag_end)
end

    		
  





    
      

  

  
    #accept_raw_tag(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


261
262
263
264
265
266
267
268
269

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 261

def accept_raw_tag(limit)
  # Let the parser handle unexpected expression tokens.
  accept_expression(limit)
  skip?(RE_TRIVIA)
  accept_whitespace_control?
  @scanner.scan(RE_TAG_END)
  emit(:token_tag_end)
  emit(:token_text) if scan_until?(RE_END_RAW)
end

    		
  





    
      

  

  
    #accept_string_literal(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 359

def accept_string_literal(limit)
  quote = @scanner.get_byte || raise
  byte = quote.ord
  double = byte == 34
  kind = double ? :token_double_quote : :token_single_quote #: t_token_kind
  emit(kind)

  if @scanner.peek_byte == byte # steep:ignore
    # Empty string
    @scanner.pos += 1
    emit(kind)
    return
  end

  # Jump to the next quote or limit, whichever is closer.
  index_ = @scanner.string.byteindex(quote, @scanner.pos) || limit
  index_ = limit if index_ > limit
  @scanner.pos = index_
  emit(double ? :token_double_quoted : :token_single_quoted)

  if @scanner.peek_byte == byte # steep:ignore
    @scanner.pos += 1
    emit(kind)
  end
end

    		
  





    
      

  

  
    #accept_tag(limit)  ⇒ void 
  

  

  


  

    
This method returns an undefined value.
(Integer) -> void


  





  
Parameters:


    
  
    
  • 
      
        limit
      
      
        (Integer)
      
      
      
    
    • 
  






  

    
      


120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 120

def accept_tag(limit)
  tag_name = @scanner.scan(RE_TAG_NAME)

  emit(:token_tag_name) unless tag_name.nil?

  case tag_name
  when "#"
    accept_inline_comment(limit)
  when "comment"
    accept_block_comment(limit)
  when "doc"
    accept_doc_comment(limit)
  when "raw"
    accept_raw_tag(limit)
  when "liquid"
    accept_line_statements(limit)
  else
    accept_expression(limit)
    skip?(RE_TRIVIA)
    accept_whitespace_control?
    @scanner.scan(RE_TAG_END)
    emit(:token_tag_end)
  end
end

    		
  





    
      

  

  
    #accept_whitespace_control?  ⇒ Boolean 
  

  

  


  

    
() -> bool


  





  

Returns:


    
  
    
  • 
      
      
        (Boolean)
      
      
      
    
    • 
  





  

    
      


386
387
388
389
390
391
392
393
394

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 386

def accept_whitespace_control?
  if @scanner.peek_byte == 45 # steep:ignore
    @scanner.pos += 1
    emit(:token_wc)
    true
  else
    false
  end
end

    		
  





    
      

  

  
    #scan_markup  ⇒ Symbol? 
  

  

  


  

    
() -> Symbol?


  





  

Returns:


    
  
    
  • 
      
      
        (Symbol, nil)
      
      
      
    
    • 
  





  

    
      


60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

    		
    
      
# File 'lib/luoma/lexer_legacy.rb', line 60

def scan_markup
  limit = nil #: Integer?

  loop do
    case @scanner.scan(RE_MARKUP)
    when "{{"
      # Output statements can be closed by `}}`, `}` or `%}`.
      # Markup delimiters are greedy and not string literal aware.
      limit = index(RE_OUT_END)

      if limit.nil?
        # Not markup and no more '}'. Emit text to end of string.
        @scanner.terminate
        emit(:token_text)
        return nil
      end

      emit(:token_out_start)
      accept_whitespace_control?
      accept_expression(limit)
      skip?(RE_TRIVIA)
      accept_whitespace_control?
      @scanner.scan(RE_OUT_END)
      emit(:token_out_end)
    when "{%"
      # Tags must be closed by `%}`.
      # Markup delimiters are greedy and not string literal aware.
      limit = index(RE_TAG_END)

      if limit.nil?
        # No more `%}`, but there could be `{{` and `}}`
        if scan_until?(RE_MARKUP)
          emit(:token_text)
        else
          # No more markup. Emit text to end of string.
          @scanner.terminate
          emit(:token_text) if @start < @scanner.pos
          return nil
        end
      else
        emit(:token_tag_start)
        accept_whitespace_control?
        skip?(RE_TRIVIA)
        accept_tag(limit)
      end
    else
      # No more `%}`, but there could be `{{` and `}}`
      if scan_until?(RE_MARKUP)
        emit(:token_text)
      else
        # No more markup. Emit text to end of string.
        @scanner.terminate
        emit(:token_text) if @start < @scanner.pos
        return nil
      end
    end
  end
end