Class: StaticEmbeddings::Converter

Inherits:
Object
  • Object
show all
Defined in:
lib/static_embeddings/converter.rb

Constant Summary collapse

REFERENCE_IMPL = 
"model2vec.StaticModel"
REFERENCE_MAX_TOKENS = 
512
ALLOWED_NORMALIZER_KEYS = 
%w[type clean_text handle_chinese_chars strip_accents lowercase].freeze
STANDARD_SPECIAL_TOKENS = 
["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"].freeze
SOURCE_FILES = 
%w[tokenizer.json config.json tokenizer_config.json model.safetensors].freeze
TOKENIZER_PROFILE = 
"BERT_WORDPIECE_V1"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source_dir) ⇒ Converter

Returns a new instance of Converter.



16
17
18
19
 
# File 'lib/static_embeddings/converter.rb', line 16

def initialize(source_dir)
  @source_dir = source_dir
  @report = {}
end

Instance Attribute Details

#report ⇒ Object (readonly)

Returns the value of attribute report.



14
15
16
 
# File 'lib/static_embeddings/converter.rb', line 14

def report
  @report
end

#source_dir ⇒ Object (readonly)

Returns the value of attribute source_dir.



14
15
16
 
# File 'lib/static_embeddings/converter.rb', line 14

def source_dir
  @source_dir
end

Instance Method Details

#convert(output_path:, model_id: nil, max_tokens: REFERENCE_MAX_TOKENS) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/static_embeddings/converter.rb', line 21

def convert(output_path:, model_id: nil, max_tokens: REFERENCE_MAX_TOKENS)
  source = load_source
  profile = audit_tokenizer(source[:tokenizer], source[:tokenizer_config])
  tokens = extract_vocab(source[:tokenizer])
  matrix, dim = extract_matrix(tokens.length)
  meta = runtime_meta(source[:config], profile, tokens, max_tokens)

  result = write_model(output_path, meta, tokens, matrix,
                       provenance_for(model_id, meta, profile, source[:config], tokens.length, dim))
  @report = result.merge(vocab_size: tokens.length, dim: dim, provenance: JSON.parse(result[:provenance]))
end