Class: Lkml::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/lkml/lexer.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(text) ⇒ Lexer

Returns a new instance of Lexer.



10
11
12
13
14
15
 
# File 'lib/lkml/lexer.rb', line 10

def initialize(text)
  @text = "#{text}\0"
  @index = 0
  @tokens = []
  @line_number = 1
end

Instance Attribute Details

#index ⇒ Object

Returns the value of attribute index.



8
9
10
 
# File 'lib/lkml/lexer.rb', line 8

def index
  @index
end

#line_number ⇒ Object

Returns the value of attribute line_number.



8
9
10
 
# File 'lib/lkml/lexer.rb', line 8

def line_number
  @line_number
end

#text ⇒ Object

Returns the value of attribute text.



8
9
10
 
# File 'lib/lkml/lexer.rb', line 8

def text
  @text
end

#tokens ⇒ Object

Returns the value of attribute tokens.



8
9
10
 
# File 'lib/lkml/lexer.rb', line 8

def tokens
  @tokens
end

Class Method Details

.expression_block?(string) ⇒ Boolean

Returns:

  • (Boolean) 


72
73
74
 
# File 'lib/lkml/lexer.rb', line 72

def self.expression_block?(string)
  Keys::EXPR_BLOCK_KEYS.any? { |key| string.start_with?("#{key}:") }
end

Instance Method Details

#advance(length = 1) ⇒ Object 



25
26
27
28
 
# File 'lib/lkml/lexer.rb', line 25

def advance(length = 1)
  @index += length
  nil
end

#consume ⇒ Object 



30
31
32
33
 
# File 'lib/lkml/lexer.rb', line 30

def consume
  advance
  @text[@index - 1]
end

#expression_block?(string) ⇒ Boolean

Returns:

  • (Boolean) 


76
77
78
 
# File 'lib/lkml/lexer.rb', line 76

def expression_block?(string)
  self.class.expression_block?(string)
end

#peek ⇒ Object 



17
18
19
 
# File 'lib/lkml/lexer.rb', line 17

def peek
  @text[@index]
end

#peek_multiple(length) ⇒ Object 



21
22
23
 
# File 'lib/lkml/lexer.rb', line 21

def peek_multiple(length)
  @text[@index, length] || ""
end

#scan ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/lkml/lexer.rb', line 35

def scan
  @tokens << Tokens::StreamStartToken.new(@line_number)
  loop do
    ch = peek
    if ch == "\0"
      @tokens << Keys::CHARACTER_TO_TOKEN[ch].new(@line_number)
      break
    elsif "\n\t ".include?(ch)
      @tokens << scan_whitespace
    elsif ch == "#"
      advance
      @tokens << scan_comment
    elsif ch == ";"
      if peek_multiple(2) == ";;"
        advance(2)
      else
        advance
      end
      @tokens << Keys::CHARACTER_TO_TOKEN[ch].new(@line_number)
    elsif ch == '"'
      advance
      @tokens << scan_quoted_literal
    elsif Keys::CHARACTER_TO_TOKEN.key?(ch)
      advance
      @tokens << Keys::CHARACTER_TO_TOKEN[ch].new(@line_number)
    elsif expression_block?(peek_multiple(25))
      @tokens << scan_literal
      advance
      @tokens << Tokens::ValueToken.new(@line_number)
      @tokens << scan_expression_block
    else
      @tokens << scan_literal
    end
  end
  @tokens.freeze
end

#scan_comment ⇒ Object 



99
100
101
102
103
 
# File 'lib/lkml/lexer.rb', line 99

def scan_comment
  chars = "#"
  chars += consume until ["\0", "\n"].include?(peek)
  Tokens::CommentToken.new(chars, @line_number)
end

#scan_expression_block ⇒ Object 



105
106
107
108
109
110
111
112
 
# File 'lib/lkml/lexer.rb', line 105

def scan_expression_block
  chars = ""
  until peek_multiple(2) == ";;"
    @line_number += 1 if peek == "\n"
    chars += consume
  end
  Tokens::ExpressionBlockToken.new(chars, @line_number)
end

#scan_literal ⇒ Object 



114
115
116
117
118
 
# File 'lib/lkml/lexer.rb', line 114

def scan_literal
  chars = ""
  chars += consume until ["\0", " ", "\n", "\t", ":", "}", "{", ",", "]"].include?(peek)
  Tokens::LiteralToken.new(chars, @line_number)
end

#scan_quoted_literal ⇒ Object 



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/lkml/lexer.rb', line 120

def scan_quoted_literal
  chars = ""
  loop do
    ch = peek
    case ch
    when '"'
      break
    when "\\"
      chars += consume
    when "\n"
      @line_number += 1
    end
    chars += consume
  end
  advance
  Tokens::QuotedLiteralToken.new(chars, @line_number)
end

#scan_whitespace ⇒ Object 



80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/lkml/lexer.rb', line 80

def scan_whitespace
  chars = ""
  next_char = peek
  while ["\n", "\t", " "].include?(next_char)
    if next_char == "\n"
      while next_char == "\n"
        chars += consume
        @line_number += 1
        next_char = peek
      end
      return Tokens::LinebreakToken.new(chars, @line_number)
    else
      chars += consume
      next_char = peek
    end
  end
  Tokens::InlineWhitespaceToken.new(chars, @line_number)
end