Class: RubyLLM::Tokenizer::Registry

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/tokenizer/registry.rb

Defined Under Namespace

Classes: Entry

Constant Summary collapse

DEFAULTS_PATH = 
File.join(File.dirname(__FILE__), "models.yml")

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Registry

Returns a new instance of Registry.



28
29
30
31
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 28

def initialize
  @entries = []
  @cache = {}
end

Class Method Details

.load_default ⇒ Object 



24
25
26
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 24

def self.load_default
  new.tap { |r| r.load_defaults_from(DEFAULTS_PATH) }
end

.parse_match(value) ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 65

def self.parse_match(value)
  if value.is_a?(String) && (md = value.match(%r{\A/(.*)/([imx]*)\z}))
    flags = 0
    flags |= Regexp::IGNORECASE if md[2].include?("i")
    flags |= Regexp::MULTILINE  if md[2].include?("m")
    flags |= Regexp::EXTENDED   if md[2].include?("x")
    Regexp.new(md[1], flags)
  else
    value
  end
end

Instance Method Details

#clear ⇒ Object 



52
53
54
55
56
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 52

def clear
  @entries.clear
  @cache.clear
  self
end

#entries ⇒ Object 



48
49
50
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 48

def entries
  @entries.dup
end

#load_defaults_from(path) ⇒ Object 



58
59
60
61
62
63
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 58

def load_defaults_from(path)
  data = YAML.load_file(path) || []
  data.each { |entry| @entries.push(build_entry(entry)) }
  @cache.clear
  self
end

#register(match:, backend:, **options) ⇒ Object

User registrations take precedence over built-ins.



34
35
36
37
38
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 34

def register(match:, backend:, **options)
  @entries.unshift(Entry.new(match: match, backend: backend.to_sym, options: options))
  @cache.clear
  self
end

#resolve(model) ⇒ Object 



40
41
42
43
44
45
46
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 40

def resolve(model)
  key = model.to_s
  return @cache[key] if @cache.key?(key)

  entry = @entries.find { |e| e.matches?(model) }
  @cache[key] = entry && build_backend(entry, key)
end