Module: MilkTea::Lexer::CharacterClasses

Included in:
MilkTea::Lexer
Defined in:
lib/milk_tea/core/lexer/character_classes.rb

Overview

Identifier lexing plus the raw-byte character-classification helpers.

Instance Method Summary collapse

Instance Method Details

#digit?(char) ⇒ Boolean

Returns:

  • (Boolean) 


52
53
54
55
 
# File 'lib/milk_tea/core/lexer/character_classes.rb', line 52

def digit?(char)
  byte = char && char.getbyte(0)
  byte ? DIGIT_BYTE[byte] : false
end

#identifier_part?(char) ⇒ Boolean

Returns:

  • (Boolean) 


39
40
41
42
 
# File 'lib/milk_tea/core/lexer/character_classes.rb', line 39

def identifier_part?(char)
  byte = char && char.getbyte(0)
  byte ? IDENT_PART_BYTE[byte] : false
end

#identifier_start?(char) ⇒ Boolean

Returns:

  • (Boolean) 


34
35
36
37
 
# File 'lib/milk_tea/core/lexer/character_classes.rb', line 34

def identifier_start?(char)
  byte = char && char.getbyte(0)
  byte ? IDENT_START_BYTE[byte] : false
end

#identifier_start_token(line, start_index) ⇒ Object 



44
45
46
47
48
49
50
 
# File 'lib/milk_tea/core/lexer/character_classes.rb', line 44

def identifier_start_token(line, start_index)
  return "" unless start_index < line.length && identifier_start?(line[start_index])

  finish = start_index + 1
  finish += 1 while finish < line.length && identifier_part?(line[finish])
  line[start_index...finish]
end

#leading_space_count(line) ⇒ Object 



28
29
30
31
32
 
# File 'lib/milk_tea/core/lexer/character_classes.rb', line 28

def leading_space_count(line)
  index = 0
  index += 1 while line.getbyte(index) == SPACE_BYTE
  index
end

#lex_identifier(line, index, line_number, line_offset:) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/milk_tea/core/lexer/character_classes.rb', line 7

def lex_identifier(line, index, line_number, line_offset:)
  start = index
  length = line.length
  index += 1
  while index < length && (byte = line.getbyte(index)) && IDENT_PART_BYTE[byte]
    index += 1
  end

  lexeme = line[start...index]
  type = Token::KEYWORDS.fetch(lexeme, :identifier)
  literal = case type
    when :true then true
    when :false then false
    when :null then nil
    else nil
  end

  @tokens << token(type, lexeme, literal, line_number, start + 1, start_offset: line_offset + start, end_offset: line_offset + index)
  index
end