Class: OpenUSD::Format::Usda::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/openusd/format/usda/lexer.rb

Overview

Converts USDA text into location-aware tokens.

Defined Under Namespace

Classes: Token

Constant Summary collapse

SYMBOLS =

Single-character grammar symbols.

"(){}[]=,:."
NUMERIC_LITERAL =

Numeric literal pattern shared by ordinary and fast-path scans.

/[+-]?(?:(?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?|inf|nan)/
ESCAPES =

Supported string escape translations.

{
  "n" => "\n", "r" => "\r", "t" => "\t", "b" => "\b",
  "f" => "\f", "\\" => "\\", "\"" => "\""
}.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, file: nil) ⇒ Lexer

Returns a new instance of Lexer.



25
26
27
28
29
30
 
# File 'lib/openusd/format/usda/lexer.rb', line 25

def initialize(source, file: nil)
  @scanner = StringScanner.new(String(source).delete_prefix("\uFEFF"))
  @file = file
  @line = 1
  @column = 1
end

Instance Attribute Details

#file ⇒ Object (readonly)

Returns the value of attribute file.



23
24
25
 
# File 'lib/openusd/format/usda/lexer.rb', line 23

def file
  @file
end

Instance Method Details

#each_token ⇒ Object

Iterate through all tokens, including the final EOF token.



33
34
35
36
37
38
39
40
41
 
# File 'lib/openusd/format/usda/lexer.rb', line 33

def each_token
  return enum_for(__method__) unless block_given?

  loop do
    token = next_token
    yield token
    break if token.type == :eof
  end
end

#next_token ⇒ Token

Returns next token from the source.

Returns:

  • (Token) —

    next token from the source



44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/openusd/format/usda/lexer.rb', line 44

def next_token
  skip_ignored
  return token(:eof, nil, "") if @scanner.eos?

  return scan_magic if @scanner.match?(/#usda\b/)
  return scan_quoted("\"\"\"", :string) if @scanner.peek(3) == "\"\"\""
  return scan_quoted("\"", :string) if @scanner.peek(1) == "\""
  return scan_quoted("@@@", :asset) if @scanner.peek(3) == "@@@"
  return scan_quoted("@", :asset) if @scanner.peek(1) == "@"
  return scan_path if @scanner.peek(1) == "<"

  scan_bare_token
end

#scan_numeric_array(component_count: nil) ⇒ Array?

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.

Scan a flat or tuple-grouped numeric array without allocating tokens.

Returns:

  • (Array, nil) —

    values, or nil when the input needs normal lexing



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/openusd/format/usda/lexer.rb', line 61

def scan_numeric_array(component_count: nil)
  rest = @scanner.rest
  closing = rest.index("]")
  return unless closing

  body = rest[0...closing]
  literals = body.scan(NUMERIC_LITERAL)
  residue = body.gsub(NUMERIC_LITERAL, "").gsub(/[\s,()]/, "")
  return unless residue.empty?
  return if component_count && (literals.length % component_count).nonzero?

  raw = rest[0..closing]
  @scanner.pos += raw.bytesize
  advance(raw)
  values = literals.map { |literal| numeric_value(literal) }
  component_count ? values.each_slice(component_count).to_a : values
end