Class: SportDb::Lexer::Tokens

Inherits:
Object
  • Object
show all
Defined in:
lib/sportdb/parser/lexer.rb

Overview

token iter/find better name

e.g. TokenBuffer/Scanner or such ??

Instance Method Summary collapse

Constructor Details

#initialize(tokens) ⇒ Tokens

Returns a new instance of Tokens.



42
43
44
45
 
# File 'lib/sportdb/parser/lexer.rb', line 42

def initialize( tokens )
    @tokens = tokens
    @pos = 0
end

Instance Method Details

#collect(&blk) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/sportdb/parser/lexer.rb', line 107

def collect( &blk )
    tokens = []
    loop do
      break if eos?
      tokens <<  if block_given?
                    blk.call( self.next )
                 else
                    self.next
                 end
    end
    tokens
end

#cur ⇒ Object

return token type (e.g. :TEXT, :NUM, etc.)



74
 
# File 'lib/sportdb/parser/lexer.rb', line 74

def cur()           peek(0); end

#eos? ⇒ Boolean

Returns:

  • (Boolean) 


48
 
# File 'lib/sportdb/parser/lexer.rb', line 48

def eos?() @pos >= @tokens.size; end

#include?(*types) ⇒ Boolean

Returns:

  • (Boolean) 


51
52
53
54
55
56
57
58
59
 
# File 'lib/sportdb/parser/lexer.rb', line 51

def include?( *types )
    pos = @pos
    ## puts "  starting include? #{types.inspect} @ #{pos}"
    while pos < @tokens.size do
        return true   if types.include?( @tokens[pos][0] )
        pos +=1
    end
    false
end

#match?(*pattern) ⇒ Boolean

pattern e.g. [:TEXT, [:VS,:SCORE], :TEXT]

Returns:

  • (Boolean) 


62
63
64
65
66
67
68
69
70
 
# File 'lib/sportdb/parser/lexer.rb', line 62

def match?( *pattern )
    ## puts "  starting match? #{pattern.inspect} @ #{@pos}"
    pattern.each_with_index do |types,offset|
        ## if single symbol wrap in array
        types = types.is_a?(Array) ? types : [types]
        return false  unless types.include?( peek(offset) )
    end
    true
end

#next ⇒ Object

note - returns complete token



96
97
98
99
100
101
102
103
104
105
 
# File 'lib/sportdb/parser/lexer.rb', line 96

def next
   # if @pos >= @tokens.size
   #     raise ArgumentError, "end of array - #{@pos} >= #{@tokens.size}"
   # end
   #   throw (standard) end of iteration here why? why not?

    t = @tokens[@pos]
    @pos += 1
    t
end

#peek(offset = 1) ⇒ Object 



86
87
88
89
90
91
92
93
 
# File 'lib/sportdb/parser/lexer.rb', line 86

def peek(offset=1)
    ## return nil if eos
    if @pos+offset >= @tokens.size
        nil
    else
       @tokens[@pos+offset][0]
    end
end

#pos ⇒ Object 



47
 
# File 'lib/sportdb/parser/lexer.rb', line 47

def pos()  @pos; end

#text(offset = 0) ⇒ Object

return content (assumed to be text)



76
77
78
79
80
81
82
83
 
# File 'lib/sportdb/parser/lexer.rb', line 76

def text(offset=0)
    ## raise error - why? why not?
    ##   return nil?
    if peek( offset ) != :text
        raise ArgumentError, "text(#{offset}) - token not a text type"
    end
    @tokens[@pos+offset][1]
end