Module: Kotoshu::Embeddings

Defined in:
lib/kotoshu/embeddings.rb,
lib/kotoshu/embeddings/search.rb,
lib/kotoshu/embeddings/protocol.rb,
lib/kotoshu/embeddings/registry.rb,
lib/kotoshu/embeddings/lru_cache.rb,
lib/kotoshu/embeddings/vocabulary.rb,
lib/kotoshu/embeddings/similarity_engine.rb,
lib/kotoshu/embeddings/similarity_search.rb,
lib/kotoshu/embeddings/embedding_pipeline.rb,
lib/kotoshu/embeddings/onnx_runtime_model.rb

Defined Under Namespace

Modules: EmbeddingModelProtocol, Protocol, SimilarityEngineProtocol, VocabularyProtocol Classes: EmbeddingPipeline, LruCache, OnnxRuntimeModel, ProtocolError, Registry, Search, SimilarityEngine, SimilaritySearch, Vocabulary

Constant Summary collapse

DEFAULT_DIMENSION = 
300
MAX_VOCABULARY_SIZE = 
100_000
VERSION = 
'2.0.0'

Class Method Summary collapse

Class Method Details

.create_pipeline(vocabulary:, model:, preload: false, pre_normalize: false) ⇒ EmbeddingPipeline

Create a custom embedding pipeline

Parameters:

  • vocabulary (Vocabulary) —

    Vocabulary instance

  • model (EmbeddingModel) —

    Model instance

  • preload (Boolean) (defaults to: false) —

    Preload embeddings

  • pre_normalize (Boolean) (defaults to: false) —

    Pre-normalize vectors

Returns:



77
78
79
80
81
82
83
84
85
 
# File 'lib/kotoshu/embeddings.rb', line 77

def self.create_pipeline(vocabulary:, model:, preload: false,
                         pre_normalize: false)
  EmbeddingPipeline.new(
    vocabulary: vocabulary,
    model: model,
    preload: preload,
    pre_normalize: pre_normalize,
  )
end

.from_cache(language:, preload: false, index: :exact) ⇒ EmbeddingPipeline

Create an EmbeddingPipeline from cache

Parameters:

  • language (String) —

    ISO 639-1 language code

  • preload (Boolean) (defaults to: false) —

    Preload embeddings into memory

  • index (:exact, :auto) (defaults to: :exact) —

    Search index type

Returns:



45
46
47
48
 
# File 'lib/kotoshu/embeddings.rb', line 45

def self.from_cache(language:, preload: false, index: :exact)
  EmbeddingPipeline.from_cache(language: language, preload: preload,
                               index: index)
end

.language_supported?(language) ⇒ Boolean

Check if a language is supported

Parameters:

  • language (String) —

    ISO 639-1 language code

Returns:

  • (Boolean) 


55
56
57
58
 
# File 'lib/kotoshu/embeddings.rb', line 55

def self.language_supported?(language)
  cache = Kotoshu::Cache::ModelCache.new
  cache.available_models_for(language.to_sym).include?(:onnx)
end

.supported_languages ⇒ Array<String>

List all supported languages

Returns:

  • (Array<String>) 


64
65
66
67
 
# File 'lib/kotoshu/embeddings.rb', line 64

def self.supported_languages
  cache = Kotoshu::Cache::ModelCache.new
  cache.all_available_models[:onnx].keys.map(&:to_s)
end