Class: MiniRuby::Token

Inherits:
Object
  • Object
show all
Extended by:
T::Sig
Defined in:
lib/miniruby/token.rb

Overview

Represents a single token (word) produced by the lexer.

Constant Summary collapse

DIGITS =

String containing all valid decimal digits

'0123456789'
HEX_DIGITS =

String containing all valid hexadecimal digits

'0123456789abcdefABCDEF'
KEYWORDS =

Set of all keywords

Set[
  'false',
  'true',
  'nil',
  'if',
  'unless',
  'while',
  'return',
  'break',
  'next',
  'end',
  'else',
  'self',
]
NONE =

Represents no token, a placeholder

:none
END_OF_FILE =

Signifies that the entire string/file has been processed, there will be no more tokens

:end_of_file
ERROR =

Holds an error message, means that the string/file could not be successfully processed

:error
LPAREN =

Left parentheses ‘(`

:lparen
RPAREN =

Right parentheses ‘)`

:rparen
COMMA =

Comma ‘,`

:comma
DOT =

Dot ‘.`

:dot
SEMICOLON =

Semicolon ‘;`

:semicolon
NEWLINE =

Newline

:newline
EQUAL =

Equal ‘=`

:equal
BANG =

Bang ‘!`

:bang
EQUAL_EQUAL =

Equal ‘==`

:equal_equal
NOT_EQUAL =

Equal ‘!=`

:not_equal
GREATER =

Greater than ‘>`

:greater
GREATER_EQUAL =

Greater equal ‘>=`

:greater_equal
LESS =

Less than ‘<`

:less
LESS_EQUAL =

Less equal ‘<=`

:less_equal
AND_AND =

Logical and ‘&&`

:and_and
AND =

Bitwise and ‘&`

:and
OR_OR =

Logical or ‘||`

:or_or
OR =

Bitwise or ‘|`

:or
QUESTION_QUESTION =

Nil coalescence ‘??`

:question_question
PLUS =

Plus ‘+`

:plus
MINUS =

Minus ‘-`

:minus
STAR =

Star ‘*`

:star
SLASH =

Slash ‘/`

:slash
INTEGER =

Integer literal eg. ‘123`

:integer
FLOAT =

Float literal eg. ‘12.3`

:float
STRING =

String literal eg. ‘“foo”`

:string
IDENTIFIER =

Identifier eg. ‘foo`

:identifier
FALSE =

Keyword ‘false`

:false
TRUE =

Keyword ‘true`

:true
NIL =

Keyword ‘nil`

:nil
IF =

Keyword ‘if`

:if
UNLESS =

Keyword ‘unless`

:unless
WHILE =

Keyword ‘while`

:while
RETURN =

Keyword ‘return`

:return
BREAK =

Keyword ‘break`

:break
NEXT =

Keyword ‘next`

:next
END_K =

Keyword ‘end`

:end
ELSE =

Keyword ‘else`

:else
SELF =

Keyword ‘self`

:self

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(type, span, value = nil) ⇒ Token

: (Symbol type, Span span, ?String? value) -> void



96
97
98
99
100
 
# File 'lib/miniruby/token.rb', line 96

def initialize(type, span, value = nil)
  @type = type
  @span = span
  @value = value
end

Instance Attribute Details

#span ⇒ Object (readonly)

: Span



93
94
95
 
# File 'lib/miniruby/token.rb', line 93

def span
  @span
end

#type ⇒ Object (readonly)

: Symbol



87
88
89
 
# File 'lib/miniruby/token.rb', line 87

def type
  @type
end

#value ⇒ Object (readonly)

: String?



90
91
92
 
# File 'lib/miniruby/token.rb', line 90

def value
  @value
end

Class Method Details

.type_to_string(type) ⇒ Object

Converts a token type into a human-readable string. : (Symbol type) -> String



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/miniruby/token.rb', line 15

def type_to_string(type)
  case type
  when NONE
    'NONE'
  when END_OF_FILE
    'END_OF_FILE'
  when ERROR
    'ERROR'
  when LPAREN
    '('
  when RPAREN
    ')'
  when COMMA
    ','
  when DOT
    '.'
  when SEMICOLON
    ';'
  when NEWLINE
    'NEWLINE'
  when EQUAL
    '='
  when BANG
    '!'
  when EQUAL_EQUAL
    '=='
  when NOT_EQUAL
    '!='
  when GREATER
    '>'
  when GREATER_EQUAL
    '>='
  when LESS
    '<'
  when LESS_EQUAL
    '<='
  when AND_AND
    '&&'
  when AND
    '&'
  when OR_OR
    '||'
  when OR
    '|'
  when QUESTION_QUESTION
    '??'
  when PLUS
    '+'
  when MINUS
    '-'
  when STAR
    '*'
  when SLASH
    '/'
  when FLOAT
    'FLOAT'
  when INTEGER
    'INTEGER'
  when STRING
    'STRING'
  when IDENTIFIER
    'IDENTIFIER'
  else
    t = type.to_s
    return t if KEYWORDS.include?(t)

    '<invalid>'
  end
end

Instance Method Details

#==(other) ⇒ Object

: (Object other) -> bool



103
104
105
106
107
 
# File 'lib/miniruby/token.rb', line 103

def ==(other)
  return false unless other.is_a?(Token)

  type == other.type && value == other.value
end

#additive_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


127
128
129
130
131
132
133
134
 
# File 'lib/miniruby/token.rb', line 127

def additive_operator?
  case @type
  when PLUS, MINUS
    true
  else
    false
  end
end

#comparison_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


147
148
149
150
151
152
153
154
 
# File 'lib/miniruby/token.rb', line 147

def comparison_operator?
  case @type
  when GREATER, GREATER_EQUAL, LESS, LESS_EQUAL
    true
  else
    false
  end
end

#equality_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


117
118
119
120
121
122
123
124
 
# File 'lib/miniruby/token.rb', line 117

def equality_operator?
  case @type
  when EQUAL_EQUAL, NOT_EQUAL
    true
  else
    false
  end
end

#inspect ⇒ Object

: -> String



110
111
112
113
114
 
# File 'lib/miniruby/token.rb', line 110

def inspect
  return "Token(#{type.inspect}, #{span.inspect})" if value.nil?

  "Token(#{type.inspect}, #{span.inspect}, #{value.inspect})"
end

#logical_and_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


177
178
179
180
181
182
183
184
 
# File 'lib/miniruby/token.rb', line 177

def logical_and_operator?
  case @type
  when AND_AND
    true
  else
    false
  end
end

#logical_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


157
158
159
160
161
162
163
164
 
# File 'lib/miniruby/token.rb', line 157

def logical_operator?
  case @type
  when OR_OR, AND_AND, QUESTION_QUESTION
    true
  else
    false
  end
end

#logical_orlike_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


167
168
169
170
171
172
173
174
 
# File 'lib/miniruby/token.rb', line 167

def logical_orlike_operator?
  case @type
  when OR_OR, QUESTION_QUESTION
    true
  else
    false
  end
end

#multiplicative_operator? ⇒ Boolean

: -> bool

Returns:

  • (Boolean) 


137
138
139
140
141
142
143
144
 
# File 'lib/miniruby/token.rb', line 137

def multiplicative_operator?
  case @type
  when STAR, SLASH
    true
  else
    false
  end
end

#to_s ⇒ Object

Converts a token into a human-readable string. : -> String



193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
 
# File 'lib/miniruby/token.rb', line 193

def to_s
  case type
  when NONE
    'NONE'
  when END_OF_FILE
    'END_OF_FILE'
  when ERROR
    "<error: #{value}>"
  when LPAREN
    '('
  when RPAREN
    ')'
  when COMMA
    ','
  when DOT
    '.'
  when SEMICOLON
    ';'
  when NEWLINE
    'NEWLINE'
  when EQUAL
    '='
  when BANG
    '!'
  when EQUAL_EQUAL
    '=='
  when NOT_EQUAL
    '!='
  when GREATER
    '>'
  when GREATER_EQUAL
    '>='
  when LESS
    '<'
  when LESS_EQUAL
    '<='
  when AND_AND
    '&&'
  when AND
    '&'
  when OR_OR
    '||'
  when OR
    '|'
  when QUESTION_QUESTION
    '??'
  when PLUS
    '+'
  when MINUS
    '-'
  when STAR
    '*'
  when SLASH
    '/'
  when FLOAT, INTEGER, IDENTIFIER
    value.to_s
  when STRING
    value.inspect #: as String
  else
    t = type.to_s
    return t if KEYWORDS.include?(t)

    '<invalid>'
  end
end

#type_name ⇒ Object

: -> String



187
188
189
 
# File 'lib/miniruby/token.rb', line 187

def type_name
  self.class.type_to_string(@type)
end