Class: Luoma::BaseLexer

Inherits:
Object
  • Object
show all
Defined in:
lib/luoma/lexer.rb,
sig/luoma/lexer.rbs

Overview

The base class for template source code tokenizers.

Direct Known Subclasses

LegacyLexer, UnifiedLexer

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(env, source) ⇒ BaseLexer

Returns a new instance of BaseLexer.

Signature:

  • (Environment, String) -> void

Parameters:



18
19
20
21
22
23
24
 
# File 'lib/luoma/lexer.rb', line 18

def initialize(env, source)
  @env = env
  @source = source
  @scanner = StringScanner.new(source)
  @start = 0
  @tokens = [] #: Array[t_token]
end

Instance Attribute Details

#tokens ⇒ Array[t_token] (readonly)

Returns the value of attribute tokens.

Returns:

  • (Array[t_token]) 


8
9
10
 
# File 'lib/luoma/lexer.rb', line 8

def tokens
  @tokens
end

Class Method Details

.tokenize(env, source) ⇒ Array[t_token]

Signature:

  • (Environment, String) -> Array[t_token]

Parameters:

Returns:

  • (Array[t_token]) 


11
12
13
14
15
 
# File 'lib/luoma/lexer.rb', line 11

def self.tokenize(env, source)
  lexer = new(env, source)
  lexer.go
  lexer.tokens
end

Instance Method Details

#emit(kind) ⇒ void

This method returns an undefined value.

(t_token_kind) -> void

Parameters:

  • kind (t_token_kind) 


44
45
46
47
48
 
# File 'lib/luoma/lexer.rb', line 44

def emit(kind)
  # @tokens << { kind: kind, start: @start, stop: @scanner.pos - 1 }
  @tokens << [kind, @start, @scanner.pos - 1]
  @start = @scanner.pos
end

#go ⇒ void

This method returns an undefined value.

Signature:

  • () -> void



27
28
29
30
 
# File 'lib/luoma/lexer.rb', line 27

def go
  state = :scan_markup
  state = send(state) until state.nil?
end

#index(pattern) ⇒ Integer?

(Regexp) -> Integer?

Parameters:

  • pattern (Regexp)

Returns:

  • (Integer, nil) 


57
58
59
60
61
62
 
# File 'lib/luoma/lexer.rb', line 57

def index(pattern)
  byte_offset = @scanner.exist?(pattern)
  return nil unless byte_offset

  @scanner.pos + byte_offset - (@scanner.matched_size || raise)
end

#scan(pattern) ⇒ String?

(Regexp) -> String?

Parameters:

  • pattern (Regexp)

Returns:

  • (String, nil) 


65
66
67
68
69
 
# File 'lib/luoma/lexer.rb', line 65

def scan(pattern)
  # NOTE: calling @scanner.scan directly yields a significant increase in
  # performance
  @scanner.scan(pattern)
end

#scan_markup ⇒ Symbol?

() -> Symbol?

Returns:

  • (Symbol, nil) 


38
39
40
 
# File 'lib/luoma/lexer.rb', line 38

def scan_markup
  raise "not implemented"
end

#scan_until?(pattern) ⇒ Boolean

(Regexp) -> bool

Parameters:

  • pattern (Regexp)

Returns:

  • (Boolean) 


73
74
75
76
77
78
79
80
81
 
# File 'lib/luoma/lexer.rb', line 73

def scan_until?(pattern)
  byte_offset = @scanner.exist?(pattern)
  if byte_offset.nil?
    false
  else
    @scanner.pos += byte_offset - (@scanner.matched_size || raise)
    true
  end
end

#skip?(pattern) ⇒ Boolean

(Regexp) -> bool

Parameters:

  • pattern (Regexp)

Returns:

  • (Boolean) 


84
85
86
87
88
89
90
91
 
# File 'lib/luoma/lexer.rb', line 84

def skip?(pattern)
  if @scanner.scan(pattern)
    @start = @scanner.pos
    true
  else
    false
  end
end

#skip_until?(pattern) ⇒ Boolean

(Regexp) -> bool

Parameters:

  • pattern (Regexp)

Returns:

  • (Boolean) 


95
96
97
98
99
100
101
102
103
104
 
# File 'lib/luoma/lexer.rb', line 95

def skip_until?(pattern)
  byte_offset = @scanner.exist?(pattern)
  if byte_offset.nil?
    false
  else
    @scanner.pos += byte_offset - (@scanner.matched_size || raise)
    @start = @scanner.pos
    true
  end
end