Module: Philiprehberger::SafeExec::Tokenizer

Defined in:
lib/philiprehberger/safe_exec/tokenizer.rb

Overview

Tokenizes expression strings into an array of typed tokens

Defined Under Namespace

Classes: Token

Constant Summary collapse

TOKEN_PATTERNS = 
[
  [:number, /\A-?\d+(?:\.\d+)?/],
  [:string, /\A'[^']*'/],
  [:string, /\A"[^"]*"/],
  [:boolean, /\A(?:true|false)\b/],
  [:null, /\Anil\b/],
  [:operator, %r{\A(?:&&|\|\||\*\*|[!=]=|>=|<=|[+\-*/%><!])}],
  [:question, /\A\?/],
  [:colon, /\A:/],
  [:lparen, /\A\(/],
  [:rparen, /\A\)/],
  [:lbracket, /\A\[/],
  [:rbracket, /\A\]/],
  [:dot, /\A\./],
  [:comma, /\A,/],
  [:identifier, /\A[a-zA-Z_][a-zA-Z0-9_]*/],
  [:whitespace, /\A\s+/]
].freeze

Class Method Summary collapse

Class Method Details

.tokenize(input) ⇒ Array<Token>

Tokenize an expression string

Parameters:

  • input (String) —

    the expression to tokenize

Returns:

  • (Array<Token>) —

    the token list

Raises:



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/philiprehberger/safe_exec/tokenizer.rb', line 33

def self.tokenize(input)
  tokens = []
  pos = 0

  while pos < input.length
    matched = false

    TOKEN_PATTERNS.each do |type, pattern|
      match = input[pos..].match(pattern)
      next unless match

      tokens << Token.new(type: type, value: match[0]) unless type == :whitespace
      pos += match[0].length
      matched = true
      break
    end

    raise Error, "unexpected character at position #{pos}: '#{input[pos]}'" unless matched
  end

  tokens
end