Class: ColumnSifter::Base::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/column_sifter/base/lexer.rb

Defined Under Namespace

Classes: Token

Constant Summary collapse

KEYWORDS = 
%w[AND OR NOT IS NULL IN].freeze
PATTERNS = 
[
  [:ws,     /\A\s+/],
  [:lparen, /\A\(/],
  [:rparen, /\A\)/],
  [:comma,  /\A,/],
  [:op,     /\A(?:<=|>=|<>|!=|=|<|>)/],
  [:number, /\A-?\d+(?:\.\d+)?/],
  [:string, /\A'(?:[^']|'')*'/],
  [:string, /\A"(?:[^"]|"")*"/],
  [:ident,  /\A[A-Za-z_][A-Za-z0-9_]*/]
].freeze

Instance Method Summary collapse

Constructor Details

#initialize(input) ⇒ Lexer

Returns a new instance of Lexer.



20
21
22
 
# File 'lib/column_sifter/base/lexer.rb', line 20

def initialize(input)
  @input = input.to_s
end

Instance Method Details

#tokenize ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/column_sifter/base/lexer.rb', line 24

def tokenize
  rest = @input
  tokens = []

  until rest.empty?
    type, matched = match_token(rest)
    raise ParseError, "unable to parse near: #{rest[0..10].inspect}" if matched.nil?

    rest = rest[matched.length..]
    next if type == :ws

    tokens << build_token(type, matched)
  end

  tokens
end