Class: RubyLLM::Tokenizer::Registry

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/tokenizer/registry.rb

Defined Under Namespace

Classes: Entry

Constant Summary collapse

DEFAULTS_PATH = 
File.join(File.dirname(__FILE__), "models.yml")

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Registry

Returns a new instance of Registry.



29
30
31
32
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 29

def initialize
  @entries = []
  @cache = {}
end

Class Method Details

.load_default ⇒ Object 



25
26
27
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 25

def self.load_default
  new.tap { |r| r.load_defaults_from(DEFAULTS_PATH) }
end

.parse_match(value) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 66

def self.parse_match(value)
  if value.is_a?(String) && (md = value.match(%r{\A/(.*)/([imx]*)\z}))
    flags = 0
    flags |= Regexp::IGNORECASE if md[2].include?("i")
    flags |= Regexp::MULTILINE  if md[2].include?("m")
    flags |= Regexp::EXTENDED   if md[2].include?("x")
    Regexp.new(md[1], flags)
  else
    value
  end
end

Instance Method Details

#clear ⇒ Object 



53
54
55
56
57
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 53

def clear
  @entries.clear
  @cache.clear
  self
end

#entries ⇒ Object 



49
50
51
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 49

def entries
  @entries.dup
end

#load_defaults_from(path) ⇒ Object 



59
60
61
62
63
64
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 59

def load_defaults_from(path)
  data = YAML.load_file(path) || []
  data.each { |entry| @entries.push(build_entry(entry)) }
  @cache.clear
  self
end

#register(match:, backend:, **options) ⇒ Object

User registrations take precedence over built-ins.



35
36
37
38
39
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 35

def register(match:, backend:, **options)
  @entries.unshift(Entry.new(match: match, backend: backend.to_sym, options: options))
  @cache.clear
  self
end

#resolve(model) ⇒ Object 



41
42
43
44
45
46
47
 
# File 'lib/ruby_llm/tokenizer/registry.rb', line 41

def resolve(model)
  key = model.to_s
  return @cache[key] if @cache.key?(key)

  entry = @entries.find { |e| e.matches?(model) }
  @cache[key] = entry && build_backend(entry, key)
end