Class: Yatte::Syntax::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/yatte/syntax/lexer.rb

Class Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Class Attribute Details

.rules ⇒ Object (readonly)

Returns the value of attribute rules.



58
59
60
 
# File 'lib/yatte/syntax/lexer.rb', line 58

def rules
  @rules
end

Class Method Details

.detect(filename) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/yatte/syntax/lexer.rb', line 19

def detect(filename)
  return :plain unless filename

  basename = File.basename(filename)

  registry.each do |tag, klass|
    klass.filenames.each do |pattern|
      return tag if File.fnmatch?(pattern, basename)
    end
  end

  :plain
end

.filenames(*patterns) ⇒ Object 



48
49
50
51
52
 
# File 'lib/yatte/syntax/lexer.rb', line 48

def filenames(*patterns)
  return @filenames if patterns.empty?

  @filenames.concat(patterns)
end

.find(tag) ⇒ Object 



15
16
17
 
# File 'lib/yatte/syntax/lexer.rb', line 15

def find(tag)
  registry[tag.to_sym]
end

.inherited(subclass) ⇒ Object 



33
34
35
36
37
 
# File 'lib/yatte/syntax/lexer.rb', line 33

def inherited(subclass)
  super
  subclass.instance_variable_set(:@rules, [])
  subclass.instance_variable_set(:@filenames, [])
end

.register(tag, klass) ⇒ Object 



11
12
13
 
# File 'lib/yatte/syntax/lexer.rb', line 11

def register(tag, klass)
  registry[tag] = klass
end

.registry ⇒ Object 



7
8
9
 
# File 'lib/yatte/syntax/lexer.rb', line 7

def registry
  @registry ||= {}
end

.rule(token_type, regex) ⇒ Object 



54
55
56
 
# File 'lib/yatte/syntax/lexer.rb', line 54

def rule(token_type, regex)
  @rules << [token_type, regex].freeze
end

.tag(name = nil) ⇒ Object

DSL methods for subclasses



41
42
43
44
45
46
 
# File 'lib/yatte/syntax/lexer.rb', line 41

def tag(name = nil)
  return @tag if name.nil?

  @tag = name.to_sym
  Lexer.register(@tag, self)
end

Instance Method Details

#tokenize(line) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/yatte/syntax/lexer.rb', line 61

def tokenize(line)
  return [[line, :plain]] if line.empty?

  rules = self.class.rules
  return [[line, :plain]] if rules.empty?

  tokens = []
  pos = 0

  while pos < line.length
    best_match = nil
    best_type = nil
    best_start = line.length

    rules.each do |type, regex|
      m = regex.match(line, pos)
      next unless m
      if m.begin(0) < best_start ||
          (m.begin(0) == best_start && (!best_match || m[0].length > best_match.length))
        best_match = m
        best_type = type
        best_start = m.begin(0)
      end
    end

    if best_match
      if best_start > pos
        tokens << [line[pos...best_start], :plain]
      end
      tokens << [best_match[0], best_type]
      pos = best_match.end(0)
    else
      tokens << [line[pos..], :plain]
      break
    end
  end

  tokens
end