Module: Tracekit::Evaluator::Lexer

Defined in:

lib/tracekit/evaluator.rb

Overview

Lexer: converts expression string into tokens.

Class Method Summary

• .tokenize(input) ⇒ Object

Class Method Details

.tokenize(input) ⇒ Object

# File 'lib/tracekit/evaluator.rb', line 164

def self.tokenize(input)
  tokens = []
  i = 0
  while i < input.length
    ch = input[i]

    # Skip whitespace
    if ch =~ /\s/
      i += 1
      next
    end

    # Two-character operators
    if i + 1 < input.length
      two = input[i, 2]
      case two
      when "=="
        tokens << Token.new(type: TokenType::EQ, value: "==")
        i += 2
        next
      when "!="
        tokens << Token.new(type: TokenType::NEQ, value: "!=")
        i += 2
        next
      when "<="
        tokens << Token.new(type: TokenType::LTE, value: "<=")
        i += 2
        next
      when ">="
        tokens << Token.new(type: TokenType::GTE, value: ">=")
        i += 2
        next
      when "&&"
        tokens << Token.new(type: TokenType::AND, value: "&&")
        i += 2
        next
      when "||"
        tokens << Token.new(type: TokenType::OR, value: "||")
        i += 2
        next
      end
    end

    # Single-character tokens
    case ch
    when "."
      tokens << Token.new(type: TokenType::DOT, value: ".")
      i += 1
      next
    when "["
      tokens << Token.new(type: TokenType::LBRACKET, value: "[")
      i += 1
      next
    when "]"
      tokens << Token.new(type: TokenType::RBRACKET, value: "]")
      i += 1
      next
    when "("
      tokens << Token.new(type: TokenType::LPAREN, value: "(")
      i += 1
      next
    when ")"
      tokens << Token.new(type: TokenType::RPAREN, value: ")")
      i += 1
      next
    when "+"
      tokens << Token.new(type: TokenType::PLUS, value: "+")
      i += 1
      next
    when "-"
      # Check if this is a negative number (unary minus before digits)
      if (tokens.empty? || [:plus, :minus, :star, :slash, :eq, :neq, :lt, :gt, :lte, :gte, :and, :or, :not, :lparen].include?(tokens.last&.type)) &&
         i + 1 < input.length && input[i + 1] =~ /\d/
        # Parse as negative number
        start = i
        i += 1
        i += 1 while i < input.length && (input[i] =~ /\d/ || input[i] == ".")
        num_str = input[start...i]
        value = num_str.include?(".") ? num_str.to_f : num_str.to_i
        tokens << Token.new(type: TokenType::NUMBER, value: value)
        next
      end
      tokens << Token.new(type: TokenType::MINUS, value: "-")
      i += 1
      next
    when "*"
      tokens << Token.new(type: TokenType::STAR, value: "*")
      i += 1
      next
    when "/"
      tokens << Token.new(type: TokenType::SLASH, value: "/")
      i += 1
      next
    when "<"
      tokens << Token.new(type: TokenType::LT, value: "<")
      i += 1
      next
    when ">"
      tokens << Token.new(type: TokenType::GT, value: ">")
      i += 1
      next
    when "!"
      tokens << Token.new(type: TokenType::NOT, value: "!")
      i += 1
      next
    end

    # String literals (double or single quoted)
    if ch == '"' || ch == "'"
      quote = ch
      i += 1
      start = i
      while i < input.length && input[i] != quote
        i += 1 if input[i] == "\\" # skip escaped char
        i += 1
      end
      tokens << Token.new(type: TokenType::STRING, value: input[start...i])
      i += 1 # skip closing quote
      next
    end

    # Numbers
    if ch =~ /\d/
      start = i
      i += 1 while i < input.length && (input[i] =~ /\d/ || input[i] == ".")
      num_str = input[start...i]
      value = num_str.include?(".") ? num_str.to_f : num_str.to_i
      tokens << Token.new(type: TokenType::NUMBER, value: value)
      next
    end

    # Identifiers and keywords
    if ch =~ /[a-zA-Z_]/
      start = i
      i += 1 while i < input.length && input[i] =~ /[a-zA-Z0-9_]/
      word = input[start...i]
      case word
      when "true"
        tokens << Token.new(type: TokenType::BOOL, value: true)
      when "false"
        tokens << Token.new(type: TokenType::BOOL, value: false)
      when "nil", "null", "None"
        tokens << Token.new(type: TokenType::NIL, value: nil)
      when "in"
        tokens << Token.new(type: TokenType::IN, value: "in")
      else
        tokens << Token.new(type: TokenType::IDENT, value: word)
      end
      next
    end

    raise "unexpected character: #{ch}"
  end

  tokens << Token.new(type: TokenType::EOF, value: nil)
  tokens
end