Class: Xberg::ChunkSizingTokenizer

Inherits:
Data
  • Object
show all
Extended by:
T::Sig
Includes:
ChunkSizing
Defined in:
lib/xberg/native.rb

Overview

Size measured in tokens from a HuggingFace tokenizer or a registered tokenizer backend.

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#cache_dir ⇒ Object (readonly)

rubocop:disable Lint/UselessMethodDefinition



381
382
383
 
# File 'lib/xberg/native.rb', line 381

def cache_dir
  @cache_dir
end

#model ⇒ Object (readonly)

rubocop:disable Lint/UselessMethodDefinition



381
382
383
 
# File 'lib/xberg/native.rb', line 381

def model
  @model
end

Class Method Details

.from_hash(hash) ⇒ Object 



402
403
404
 
# File 'lib/xberg/native.rb', line 402

def self.from_hash(hash)
  new(model: hash[:model] || hash["model"], cache_dir: hash[:cache_dir] || hash["cache_dir"])
end

Instance Method Details

#characters? ⇒ Boolean

Returns:

  • (Boolean) 


396
 
# File 'lib/xberg/native.rb', line 396

def characters? = false

#tokenizer? ⇒ Boolean

Returns:

  • (Boolean) 


398
399
400
 
# File 'lib/xberg/native.rb', line 398

def tokenizer? = true
# @param hash [Hash] deserialized from the native extension
# @return [self]