Class: MiniRuby::Token

Inherits:
Object
  • Object
show all
Extended by:
T::Sig
Defined in:
lib/miniruby/token.rb

Overview

Represents a single token (word) produced by the lexer.

Constant Summary collapse

DIGITS =

String containing all valid decimal digits

'0123456789'
HEX_DIGITS =

String containing all valid hexadecimal digits

'0123456789abcdefABCDEF'
KEYWORDS =

Set of all keywords

Set[
  'false',
  'true',
  'nil',
  'if',
  'unless',
  'while',
  'return',
  'break',
  'next',
  'end',
  'else',
  'self',
]
NONE =

Represents no token, a placeholder

:none
END_OF_FILE =

Signifies that the entire string/file has been processed, there will be no more tokens

:end_of_file
ERROR =

Holds an error message, means that the string/file could not be successfully processed

:error
LPAREN =

Left parentheses ‘(`

:lparen
RPAREN =

Right parentheses ‘)`

:rparen
COMMA =

Comma ‘,`

:comma
DOT =

Dot ‘.`

:dot
SEMICOLON =

Semicolon ‘;`

:semicolon
NEWLINE =

Newline

:newline
EQUAL =

Equal ‘=`

:equal
BANG =

Bang ‘!`

:bang
EQUAL_EQUAL =

Equal ‘==`

:equal_equal
NOT_EQUAL =

Equal ‘!=`

:not_equal
GREATER =

Greater than ‘>`

:greater
GREATER_EQUAL =

Greater equal ‘>=`

:greater_equal
LESS =

Less than ‘<`

:less
LESS_EQUAL =

Less equal ‘<=`

:less_equal
PLUS =

Plus ‘+`

:plus
MINUS =

Minus ‘-`

:minus
STAR =

Star ‘*`

:star
SLASH =

Slash ‘/`

:slash
INTEGER =

Integer literal eg. ‘123`

:integer
FLOAT =

Float literal eg. ‘12.3`

:float
STRING =

String literal eg. ‘“foo”`

:string
IDENTIFIER =

Identifier eg. ‘foo`

:identifier
FALSE =

Keyword ‘false`

:false
TRUE =

Keyword ‘true`

:true
NIL =

Keyword ‘nil`

:nil
IF =

Keyword ‘if`

:if
UNLESS =

Keyword ‘unless`

:unless
WHILE =

Keyword ‘while`

:while
RETURN =

Keyword ‘return`

:return
BREAK =

Keyword ‘break`

:break
NEXT =

Keyword ‘next`

:next
END_K =

Keyword ‘end`

:end
ELSE =

Keyword ‘else`

:else
SELF =

Keyword ‘self`

:self

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(type, span, value = nil) ⇒ Token

: (Symbol type, Span span, ?String? value) -> void



86
87
88
89
90
 
# File 'lib/miniruby/token.rb', line 86

def initialize(type, span, value = nil)
  @type = type
  @span = span
  @value = value
end

Instance Attribute Details

#span ⇒ Object (readonly)

: Span



83
84
85
 
# File 'lib/miniruby/token.rb', line 83

def span
  @span
end

#type ⇒ Object (readonly)

: Symbol



77
78
79
 
# File 'lib/miniruby/token.rb', line 77

def type
  @type
end

#value ⇒ Object (readonly)

: String?



80
81
82
 
# File 'lib/miniruby/token.rb', line 80

def value
  @value
end

Class Method Details

.type_to_string(type) ⇒ Object

Converts a token type into a human-readable string. : (Symbol type) -> String



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/miniruby/token.rb', line 15

def type_to_string(type)
  case type
  when NONE
    'NONE'
  when END_OF_FILE
    'END_OF_FILE'
  when ERROR
    'ERROR'
  when LPAREN
    '('
  when RPAREN
    ')'
  when COMMA
    ','
  when DOT
    '.'
  when SEMICOLON
    ';'
  when NEWLINE
    'NEWLINE'
  when EQUAL
    '='
  when BANG
    '!'
  when EQUAL_EQUAL
    '=='
  when NOT_EQUAL
    '!='
  when GREATER
    '>'
  when GREATER_EQUAL
    '>='
  when LESS
    '<'
  when LESS_EQUAL
    '<='
  when PLUS
    '+'
  when MINUS
    '-'
  when STAR
    '*'
  when SLASH
    '/'
  when FLOAT
    'FLOAT'
  when INTEGER
    'INTEGER'
  when STRING
    'STRING'
  when IDENTIFIER
    'IDENTIFIER'
  else
    t = type.to_s
    return t if KEYWORDS.include?(t)

    '<invalid>'
  end
end

Instance Method Details

#==(other) ⇒ Object

: (Object other) -> bool



93
94
95
96
97
 
# File 'lib/miniruby/token.rb', line 93

def ==(other)
  return false unless other.is_a?(Token)

  type == other.type && value == other.value
end

#additive_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


117
118
119
120
121
122
123
124
 
# File 'lib/miniruby/token.rb', line 117

def additive_operator?
  case @type
  when PLUS, MINUS
    true
  else
    false
  end
end

#comparison_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


137
138
139
140
141
142
143
144
 
# File 'lib/miniruby/token.rb', line 137

def comparison_operator?
  case @type
  when GREATER, GREATER_EQUAL, LESS, LESS_EQUAL
    true
  else
    false
  end
end

#equality_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


107
108
109
110
111
112
113
114
 
# File 'lib/miniruby/token.rb', line 107

def equality_operator?
  case @type
  when EQUAL_EQUAL, NOT_EQUAL
    true
  else
    false
  end
end

#inspect ⇒ Object

: -> String



100
101
102
103
104
 
# File 'lib/miniruby/token.rb', line 100

def inspect
  return "Token(#{type.inspect}, #{span.inspect})" if value.nil?

  "Token(#{type.inspect}, #{span.inspect}, #{value.inspect})"
end

#multiplicative_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


127
128
129
130
131
132
133
134
 
# File 'lib/miniruby/token.rb', line 127

def multiplicative_operator?
  case @type
  when STAR, SLASH
    true
  else
    false
  end
end

#to_s ⇒ Object

Converts a token into a human-readable string. : -> String



153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
 
# File 'lib/miniruby/token.rb', line 153

def to_s
  case type
  when NONE
    'NONE'
  when END_OF_FILE
    'END_OF_FILE'
  when ERROR
    "<error: #{value}>"
  when LPAREN
    '('
  when RPAREN
    ')'
  when COMMA
    ','
  when DOT
    '.'
  when SEMICOLON
    ';'
  when NEWLINE
    'NEWLINE'
  when EQUAL
    '='
  when BANG
    '!'
  when EQUAL_EQUAL
    '=='
  when NOT_EQUAL
    '!='
  when GREATER
    '>'
  when GREATER_EQUAL
    '>='
  when LESS
    '<'
  when LESS_EQUAL
    '<='
  when PLUS
    '+'
  when MINUS
    '-'
  when STAR
    '*'
  when SLASH
    '/'
  when FLOAT, INTEGER, IDENTIFIER
    value.to_s
  when STRING
    value.inspect #: as String
  else
    t = type.to_s
    return t if KEYWORDS.include?(t)

    '<invalid>'
  end
end

#type_name ⇒ Object

: -> String



147
148
149
 
# File 'lib/miniruby/token.rb', line 147

def type_name
  self.class.type_to_string(@type)
end