Class: LexerKit::Core::Token

Inherits:
Object
  • Object
show all
Defined in:
lib/lexer_kit/core/token.rb

Overview

Token represents a lexed token with position and optional location info. Line and column are lazily computed to minimize overhead.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(id:, name:, start:, len:, source:, meta: nil) ⇒ Token

Returns a new instance of Token.

Parameters:

  • id (Integer) —

    token ID

  • name (Symbol) —

    token name

  • start (Integer) —

    start byte offset

  • len (Integer) —

    length in bytes

  • source (Source) —

    source object for location lookup

  • meta (Hash, nil) (defaults to: nil) —

    optional metadata for this token



16
17
18
19
20
21
22
23
24
25
 
# File 'lib/lexer_kit/core/token.rb', line 16

def initialize(id:, name:, start:, len:, source:, meta: nil)
  @id = id
  @name = name
  @start = start
  @len = len
  @source = source
  @meta = meta || {}
  @line = nil
  @col = nil
end

Instance Attribute Details

#id ⇒ Object (readonly)

Returns the value of attribute id.



8
9
10
 
# File 'lib/lexer_kit/core/token.rb', line 8

def id
  @id
end

#len ⇒ Object (readonly)

Returns the value of attribute len.



8
9
10
 
# File 'lib/lexer_kit/core/token.rb', line 8

def len
  @len
end

#meta ⇒ Object (readonly)

Returns the value of attribute meta.



8
9
10
 
# File 'lib/lexer_kit/core/token.rb', line 8

def meta
  @meta
end

#name ⇒ Object (readonly) Also known as: type

Returns the value of attribute name.



8
9
10
 
# File 'lib/lexer_kit/core/token.rb', line 8

def name
  @name
end

#start ⇒ Object (readonly)

Returns the value of attribute start.



8
9
10
 
# File 'lib/lexer_kit/core/token.rb', line 8

def start
  @start
end

Instance Method Details

#==(other) ⇒ Object 



85
86
87
88
89
 
# File 'lib/lexer_kit/core/token.rb', line 85

def ==(other)
  return false unless other.is_a?(Token)

  @name == other.name && @start == other.start && @len == other.len
end

#col ⇒ Integer

Get the column number (1-based, lazily computed)

Returns:

  • (Integer) 


45
46
47
 
# File 'lib/lexer_kit/core/token.rb', line 45

def col
  @col ||= @source.line_col(@start)[1]
end

#eql?(other) ⇒ Boolean

Returns:

  • (Boolean) 


91
92
93
 
# File 'lib/lexer_kit/core/token.rb', line 91

def eql?(other)
  self == other
end

#error? ⇒ Boolean

Check if this is an error token

Returns:

  • (Boolean) 


57
58
59
 
# File 'lib/lexer_kit/core/token.rb', line 57

def error?
  @id == LexerKit::INVALID_TOKEN_ID
end

#hash ⇒ Object 



95
96
97
 
# File 'lib/lexer_kit/core/token.rb', line 95

def hash
  [@name, @start, @len].hash
end

#inspect ⇒ Object 



115
116
117
 
# File 'lib/lexer_kit/core/token.rb', line 115

def inspect
  "#<LexerKit::Core::Token #{@name} #{@start}:#{@len} #{text.inspect}>"
end

#line ⇒ Integer

Get the line number (1-based, lazily computed)

Returns:

  • (Integer) 


39
40
41
 
# File 'lib/lexer_kit/core/token.rb', line 39

def line
  @line ||= @source.line_col(@start)[0]
end

#render_diagnostic(message, level: :error, notes: nil, color: $stdout.tty?) ⇒ String

Render diagnostic with source context

Parameters:

  • message (String) —

    error message

  • level (Symbol) (defaults to: :error) —

    :error, :warning, or :note

  • notes (Array<String>, nil) (defaults to: nil) —

    additional notes

  • color (Boolean) (defaults to: $stdout.tty?) —

    enable ANSI colors

Returns:

  • (String) 


81
82
83
 
# File 'lib/lexer_kit/core/token.rb', line 81

def render_diagnostic(message, level: :error, notes: nil, color: $stdout.tty?)
  to_diagnostic(message, level: level, notes: notes).render(@source, color: color)
end

#span ⇒ Span

Get the span object

Returns:



51
52
53
 
# File 'lib/lexer_kit/core/token.rb', line 51

def span
  @span ||= Span.new(@start, @len)
end

#text ⇒ String

Get the token text

Returns:

  • (String) 


33
34
35
 
# File 'lib/lexer_kit/core/token.rb', line 33

def text
  @source.bytes.byteslice(@start, @len)
end

#to_diagnostic(message, level: :error, notes: nil) ⇒ Diagnostic

Create a Diagnostic from this token

Parameters:

  • message (String) —

    error message

  • level (Symbol) (defaults to: :error) —

    :error, :warning, or :note

  • notes (Array<String>, nil) (defaults to: nil) —

    additional notes

Returns:



66
67
68
69
70
71
72
73
 
# File 'lib/lexer_kit/core/token.rb', line 66

def to_diagnostic(message, level: :error, notes: nil)
  Diagnostic.new(
    level: level,
    message: message,
    span: span,
    notes: notes
  )
end

#to_h ⇒ Object 



99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/lexer_kit/core/token.rb', line 99

def to_h
  {
    id: @id,
    name: @name,
    text: text,
    start: @start,
    len: @len,
    line: line,
    col: col
  }
end

#to_s ⇒ Object 



111
112
113
 
# File 'lib/lexer_kit/core/token.rb', line 111

def to_s
  "#{@name}(#{text.inspect})"
end