Module: Kotoshu
- Defined in:
- lib/kotoshu/embeddings.rb,
lib/kotoshu.rb,
lib/kotoshu.rb,
lib/kotoshu/cli.rb,
lib/kotoshu/core.rb,
lib/kotoshu/data.rb,
lib/kotoshu/cache.rb,
lib/kotoshu/paths.rb,
lib/kotoshu/models.rb,
lib/kotoshu/grammar.rb,
lib/kotoshu/plugins.rb,
lib/kotoshu/readers.rb,
lib/kotoshu/results.rb,
lib/kotoshu/version.rb,
lib/kotoshu/commands.rb,
lib/kotoshu/defaults.rb,
lib/kotoshu/keyboard.rb,
lib/kotoshu/language.rb,
lib/kotoshu/analyzers.rb,
lib/kotoshu/documents.rb,
lib/kotoshu/integrity.rb,
lib/kotoshu/languages.rb,
lib/kotoshu/algorithms.rb,
lib/kotoshu/cli/errors.rb,
lib/kotoshu/components.rb,
lib/kotoshu/debug_mode.rb,
lib/kotoshu/dictionary.rb,
lib/kotoshu/cache/cache.rb,
lib/kotoshu/suggestions.rb,
lib/kotoshu/debug_logger.rb,
lib/kotoshu/dictionaries.rb,
lib/kotoshu/grammar/rule.rb,
lib/kotoshu/spellchecker.rb,
lib/kotoshu/configuration.rb,
lib/kotoshu/cli/auto_setup.rb,
lib/kotoshu/core/trie/node.rb,
lib/kotoshu/core/trie/trie.rb,
lib/kotoshu/fluent_checker.rb,
lib/kotoshu/language/suika.rb,
lib/kotoshu/metrics_module.rb,
lib/kotoshu/models/context.rb,
lib/kotoshu/plugins/plugin.rb,
lib/kotoshu/project_config.rb,
lib/kotoshu/results/result.rb,
lib/kotoshu/string_metrics.rb,
lib/kotoshu/core/exceptions.rb,
lib/kotoshu/data_structures.rb,
lib/kotoshu/dictionary/base.rb,
lib/kotoshu/keyboard/layout.rb,
lib/kotoshu/resource_bundle.rb,
lib/kotoshu/source_registry.rb,
lib/kotoshu/cache/base_cache.rb,
lib/kotoshu/core/models/word.rb,
lib/kotoshu/plugins/registry.rb,
lib/kotoshu/readers/aff_data.rb,
lib/kotoshu/resource_manager.rb,
lib/kotoshu/algorithms/lookup.rb,
lib/kotoshu/cache/model_cache.rb,
lib/kotoshu/cli/cache_command.rb,
lib/kotoshu/cli/status_report.rb,
lib/kotoshu/core/trie/builder.rb,
lib/kotoshu/dictionary/cspell.rb,
lib/kotoshu/dictionary/custom.rb,
lib/kotoshu/embeddings/search.rb,
lib/kotoshu/keyboard/registry.rb,
lib/kotoshu/language/detector.rb,
lib/kotoshu/language/registry.rb,
lib/kotoshu/metrics_collector.rb,
lib/kotoshu/models/onnx_model.rb,
lib/kotoshu/models/suggestion.rb,
lib/kotoshu/algorithms/suggest.rb,
lib/kotoshu/cache/lookup_cache.rb,
lib/kotoshu/cli/batch_reporter.rb,
lib/kotoshu/dictionary/unified.rb,
lib/kotoshu/documents/document.rb,
lib/kotoshu/integrity/manifest.rb,
lib/kotoshu/integrity/net_http.rb,
lib/kotoshu/language/segmenter.rb,
lib/kotoshu/readers/aff_reader.rb,
lib/kotoshu/readers/dic_reader.rb,
lib/kotoshu/dictionary/hunspell.rb,
lib/kotoshu/documents/text_node.rb,
lib/kotoshu/embeddings/protocol.rb,
lib/kotoshu/embeddings/registry.rb,
lib/kotoshu/grammar/rule_engine.rb,
lib/kotoshu/grammar/rule_loader.rb,
lib/kotoshu/integrity/audit_log.rb,
lib/kotoshu/language/identifier.rb,
lib/kotoshu/personal_dictionary.rb,
lib/kotoshu/readers/file_reader.rb,
lib/kotoshu/suggestions/context.rb,
lib/kotoshu/cache/language_cache.rb,
lib/kotoshu/cli/personal_command.rb,
lib/kotoshu/components/tokenizer.rb,
lib/kotoshu/dictionaries/catalog.rb,
lib/kotoshu/embeddings/lru_cache.rb,
lib/kotoshu/suggestions/pipeline.rb,
lib/kotoshu/cache/eviction_policy.rb,
lib/kotoshu/cache/frequency_cache.rb,
lib/kotoshu/cli/display_formatter.rb,
lib/kotoshu/cli/language_resolver.rb,
lib/kotoshu/cli/progress_reporter.rb,
lib/kotoshu/components/pos_tagger.rb,
lib/kotoshu/configuration/builder.rb,
lib/kotoshu/dictionary/plain_text.rb,
lib/kotoshu/dictionary/repository.rb,
lib/kotoshu/dictionary/unix_words.rb,
lib/kotoshu/embeddings/vocabulary.rb,
lib/kotoshu/languages/ar/language.rb,
lib/kotoshu/languages/de/language.rb,
lib/kotoshu/languages/en/language.rb,
lib/kotoshu/languages/es/language.rb,
lib/kotoshu/languages/fa/language.rb,
lib/kotoshu/languages/fr/language.rb,
lib/kotoshu/languages/he/language.rb,
lib/kotoshu/languages/ja/language.rb,
lib/kotoshu/languages/pt/language.rb,
lib/kotoshu/languages/ru/language.rb,
lib/kotoshu/models/fasttext_model.rb,
lib/kotoshu/models/semantic_error.rb,
lib/kotoshu/models/word_embedding.rb,
lib/kotoshu/suggestions/generator.rb,
lib/kotoshu/cache/suggestion_cache.rb,
lib/kotoshu/cli/navigation_manager.rb,
lib/kotoshu/commands/cache_command.rb,
lib/kotoshu/commands/check_command.rb,
lib/kotoshu/commands/model_command.rb,
lib/kotoshu/components/synthesizer.rb,
lib/kotoshu/configuration/resolver.rb,
lib/kotoshu/core/models/affix_rule.rb,
lib/kotoshu/documents/source_range.rb,
lib/kotoshu/models/embedding_model.rb,
lib/kotoshu/multi_language_checker.rb,
lib/kotoshu/readers/lookup_builder.rb,
lib/kotoshu/suggestions/suggestion.rb,
lib/kotoshu/algorithms/permutations.rb,
lib/kotoshu/cli/completions_command.rb,
lib/kotoshu/core/indexed_dictionary.rb,
lib/kotoshu/keyboard/layouts/azerty.rb,
lib/kotoshu/keyboard/layouts/dvorak.rb,
lib/kotoshu/keyboard/layouts/jcuken.rb,
lib/kotoshu/keyboard/layouts/qwerty.rb,
lib/kotoshu/keyboard/layouts/qwertz.rb,
lib/kotoshu/language/languages/base.rb,
lib/kotoshu/language/tokenizer/base.rb,
lib/kotoshu/models/nearest_neighbor.rb,
lib/kotoshu/algorithms/ngram_suggest.rb,
lib/kotoshu/cli/interactive_reviewer.rb,
lib/kotoshu/components/spell_checker.rb,
lib/kotoshu/data/common_words_loader.rb,
lib/kotoshu/language/normalizer/base.rb,
lib/kotoshu/algorithms/capitalization.rb,
lib/kotoshu/algorithms/phonet_suggest.rb,
lib/kotoshu/documents/source_position.rb,
lib/kotoshu/integrity/rotation_policy.rb,
lib/kotoshu/readers/condition_checker.rb,
lib/kotoshu/language/normalizer/arabic.rb,
lib/kotoshu/language/normalizer/hebrew.rb,
lib/kotoshu/suggestions/suggestion_set.rb,
lib/kotoshu/analyzers/semantic_analyzer.rb,
lib/kotoshu/language/normalizer/persian.rb,
lib/kotoshu/data_structures/bloom_filter.rb,
lib/kotoshu/embeddings/similarity_engine.rb,
lib/kotoshu/embeddings/similarity_search.rb,
lib/kotoshu/documents/plain_text_document.rb,
lib/kotoshu/embeddings/embedding_pipeline.rb,
lib/kotoshu/embeddings/onnx_runtime_model.rb,
lib/kotoshu/spellchecker/parallel_checker.rb,
lib/kotoshu/core/models/result/word_result.rb,
lib/kotoshu/components/whitespace_tokenizer.rb,
lib/kotoshu/core/models/result/document_result.rb,
lib/kotoshu/language/tokenizer/latin_tokenizer.rb,
lib/kotoshu/language/tokenizer/french_tokenizer.rb,
lib/kotoshu/language/tokenizer/german_tokenizer.rb,
lib/kotoshu/components/passthrough_spell_checker.rb,
lib/kotoshu/language/tokenizer/russian_tokenizer.rb,
lib/kotoshu/language/tokenizer/spanish_tokenizer.rb,
lib/kotoshu/suggestions/strategies/base_strategy.rb,
lib/kotoshu/grammar/pattern_matchers/base_matcher.rb,
lib/kotoshu/language/tokenizer/japanese_tokenizer.rb,
lib/kotoshu/suggestions/strategies/ngram_strategy.rb,
lib/kotoshu/grammar/pattern_matchers/phrase_matcher.rb,
lib/kotoshu/language/tokenizer/portuguese_tokenizer.rb,
lib/kotoshu/suggestions/strategies/phonetic_strategy.rb,
lib/kotoshu/suggestions/strategies/semantic_strategy.rb,
lib/kotoshu/suggestions/strategies/symspell_strategy.rb,
lib/kotoshu/suggestions/strategies/composite_strategy.rb,
lib/kotoshu/grammar/pattern_matchers/vowel_sound_matcher.rb,
lib/kotoshu/suggestions/strategies/edit_distance_strategy.rb,
lib/kotoshu/grammar/pattern_matchers/double_negative_matcher.rb,
lib/kotoshu/suggestions/strategies/keyboard_proximity_strategy.rb,
lib/kotoshu/grammar/pattern_matchers/possessive_context_matcher.rb,
sig/kotoshu.rbs
Overview
OnnxRuntimeModel - ONNX Runtime wrapper for FastText embeddings
Provides embedding inference using ONNX Runtime. Supports single lookups, batch inference, and vocabulary-aware operations.
Defined Under Namespace
Modules: Algorithms, Analyzers, Cache, Cli, Commands, Components, Core, Data, DataStructures, Debug, Defaults, Dictionaries, Dictionary, Documents, Embeddings, Grammar, Integrity, Keyboard, Language, Languages, Metrics, Models, Paths, Plugins, Readers, Results, StringMetrics, Suggestions Classes: AffixRuleError, CacheCommand, CheckCommand, Configuration, ConfigurationError, DictionaryNotFoundError, Error, IntegrityError, InvalidDictionaryFormatError, ModelCommand, MultiLanguageChecker, PersonalDictionary, ProjectConfig, ResourceBundle, ResourceManager, ResourceNotCachedError, ResourceNotSetupError, ResourceResolutionError, SourceRegistry, SpellcheckError, Spellchecker, SuikaUnavailable
Constant Summary collapse
- VERSION =
'0.5.1'
Class Method Summary collapse
-
.check(text, language: nil, **_options) ⇒ Models::Result::DocumentResult
Check text for spelling errors.
-
.check_file(path, language: nil, **_options) ⇒ Models::Result::DocumentResult
Check a file for spelling errors.
-
.check_files(paths, **options) ⇒ Array<Models::Result::DocumentResult>
Check multiple files for spelling errors.
-
.configuration ⇒ Configuration
Get the global configuration.
-
.configuration=(config) ⇒ Configuration
Replace the global configuration.
-
.configure {|configuration| ... } ⇒ Configuration
Global configuration instance.
-
.correct?(word, language: nil) ⇒ Boolean
Check if a word is spelled correctly.
-
.detect_language(text) ⇒ String?
Detect language of text.
-
.detect_language_with_confidence(text) ⇒ Array<String, Float>
Detect language with confidence score.
-
.dictionary(source = nil) ⇒ Core::IndexedDictionary
Convenience method for creating an indexed dictionary.
-
.get_language(code) ⇒ Class?
Get language class by code.
-
.language ⇒ Module
Access the language module.
-
.language_registered?(code) ⇒ Boolean
Check if a language is registered.
-
.languages_setup ⇒ Array<String>
List languages that have been set up.
-
.misspelled?(word, language: nil) ⇒ Boolean
Check if a word is misspelled.
-
.register_dictionary_type(type, klass) ⇒ Object
Register a custom dictionary type.
-
.register_suggestion_algorithm(name, klass) ⇒ Object
Register a custom suggestion algorithm.
-
.reset_spellchecker ⇒ Object
Reset the spellchecker cache.
-
.resolve(language: nil, want: nil) ⇒ ResourceBundle
Resolve language resources from the cache (no download).
-
.setup(*languages, want: nil, **opts) ⇒ SetupResult+
Set up resources for one or more languages (download or register local files).
-
.setup?(language, resource = nil) ⇒ Boolean
Check if a language (or a specific resource for that language) is set up.
-
.spellchecker ⇒ Spellchecker
Default spellchecker (singleton).
-
.spellchecker_for(language) ⇒ Spellchecker
Get a spellchecker for a specific language (cache-only, raises on miss).
-
.suggest(word, language: nil, **options) ⇒ Suggestions::SuggestionSet
Get spelling suggestions for a word.
-
.suggestion_pipeline(*strategies) ⇒ Suggestions::Strategies::CompositeStrategy
Convenience method for creating a suggestion pipeline.
-
.supported_languages ⇒ Array<String>
Get all supported language codes.
-
.trie(source = nil) ⇒ Core::Trie::Trie
Convenience method for creating a trie.
Class Method Details
.check(text, language: nil, **_options) ⇒ Models::Result::DocumentResult
Check text for spelling errors. Hot path.
302 303 304 305 |
# File 'lib/kotoshu.rb', line 302 def self.check(text, language: nil, **) checker = language ? spellchecker_for(language) : spellchecker checker.check(text) end |
.check_file(path, language: nil, **_options) ⇒ Models::Result::DocumentResult
Check a file for spelling errors. Hot path.
319 320 321 322 |
# File 'lib/kotoshu.rb', line 319 def self.check_file(path, language: nil, **) checker = language ? spellchecker_for(language) : spellchecker checker.check_file(path) end |
.check_files(paths, **options) ⇒ Array<Models::Result::DocumentResult>
Check multiple files for spelling errors.
333 334 335 |
# File 'lib/kotoshu.rb', line 333 def self.check_files(paths, **) paths.map { |path| check_file(path, **) } end |
.configuration ⇒ Configuration
Get the global configuration.
95 96 97 |
# File 'lib/kotoshu.rb', line 95 def self.configuration Configuration.instance end |
.configuration=(config) ⇒ Configuration
Replace the global configuration. Useful for tests and for embedding kotoshu in a larger application that wants to share a Configuration across multiple Spellcheckers without each one independently touching the singleton.
The argument must be a Configuration instance (type-checked so a typo doesn't silently flip the global to nil and break every downstream lookup).
113 114 115 116 117 |
# File 'lib/kotoshu.rb', line 113 def self.configuration=(config) raise ArgumentError, "configuration must be a Configuration instance" unless config.is_a?(Configuration) Configuration.instance = config end |
.configure {|configuration| ... } ⇒ Configuration
Global configuration instance.
84 85 86 87 |
# File 'lib/kotoshu.rb', line 84 def self.configure yield configuration if block_given? configuration end |
.correct?(word, language: nil) ⇒ Boolean
Check if a word is spelled correctly. Hot path — cache-only, raises if language not set up.
258 259 260 261 |
# File 'lib/kotoshu.rb', line 258 def self.correct?(word, language: nil) checker = language ? spellchecker_for(language) : spellchecker checker.correct?(word) end |
.detect_language(text) ⇒ String?
Detect language of text.
425 426 427 |
# File 'lib/kotoshu.rb', line 425 def self.detect_language(text) Language.detect(text) end |
.detect_language_with_confidence(text) ⇒ Array<String, Float>
Detect language with confidence score.
438 439 440 |
# File 'lib/kotoshu.rb', line 438 def self.detect_language_with_confidence(text) Language.detect_with_confidence(text) end |
.dictionary(source = nil) ⇒ Core::IndexedDictionary
Convenience method for creating an indexed dictionary.
341 342 343 344 345 346 347 348 349 350 351 352 |
# File 'lib/kotoshu.rb', line 341 def self.dictionary(source = nil) case source when Array Core::IndexedDictionary.new(source) when String Core::IndexedDictionary.from_file(source) when nil, Hash Core::IndexedDictionary.new else raise ArgumentError, "Invalid dictionary source: #{source.inspect}" end end |
.get_language(code) ⇒ Class?
Get language class by code.
449 450 451 |
# File 'lib/kotoshu.rb', line 449 def self.get_language(code) Language.get(code) end |
.language ⇒ Module
Access the language module.
413 414 415 |
# File 'lib/kotoshu.rb', line 413 def self.language Language end |
.language_registered?(code) ⇒ Boolean
Check if a language is registered.
460 461 462 |
# File 'lib/kotoshu.rb', line 460 def self.language_registered?(code) Language.registered?(code) end |
.languages_setup ⇒ Array<String>
List languages that have been set up.
229 230 231 |
# File 'lib/kotoshu.rb', line 229 def self.languages_setup ResourceManager.languages_setup end |
.misspelled?(word, language: nil) ⇒ Boolean
Check if a word is misspelled. Hot path.
269 270 271 |
# File 'lib/kotoshu.rb', line 269 def self.misspelled?(word, language: nil) !correct?(word, language: language) end |
.register_dictionary_type(type, klass) ⇒ Object
Register a custom dictionary type.
392 393 394 |
# File 'lib/kotoshu.rb', line 392 def self.register_dictionary_type(type, klass) Dictionary.register_type(type, klass) end |
.register_suggestion_algorithm(name, klass) ⇒ Object
Register a custom suggestion algorithm.
403 404 405 |
# File 'lib/kotoshu.rb', line 403 def self.register_suggestion_algorithm(name, klass) Suggestions::Strategies::BaseStrategy.register_type(name, klass) end |
.reset_spellchecker ⇒ Object
Reset the spellchecker cache. The next call to spellchecker or
spellchecker_for re-resolves from the current configuration.
Does NOT eagerly reload — clearing the cache is enough. This makes the method safe to call between tests even when no language is set up yet (the next call will raise ResourceNotSetupError per the strict two-stage contract).
240 241 242 243 244 |
# File 'lib/kotoshu.rb', line 240 def self.reset_spellchecker @spellchecker = nil @spellcheckers = nil nil end |
.resolve(language: nil, want: nil) ⇒ ResourceBundle
Resolve language resources from the cache (no download).
163 164 165 166 167 168 169 |
# File 'lib/kotoshu.rb', line 163 def self.resolve(language: nil, want: nil) lang = language || configuration.default_language raise ResourceNotSetupError.new(lang || "default", "spelling") if lang.nil? want_param = want || ResourceManager::DEFAULT_WANT ResourceManager.resolve(language: lang, want: want_param) end |
.setup(*languages, want: nil, **opts) ⇒ SetupResult+
Set up resources for one or more languages (download or register local files).
Idempotent: re-running with the same args is a no-op unless force: true.
197 198 199 200 201 202 203 204 205 206 |
# File 'lib/kotoshu.rb', line 197 def self.setup(*languages, want: nil, **opts) raise ArgumentError, "Kotoshu.setup requires at least one language" if languages.empty? want_param = want || ResourceManager::DEFAULT_WANT if languages.size == 1 ResourceManager.setup(languages.first, want: want_param, **opts) else languages.map { |lang| ResourceManager.setup(lang, want: want_param, **opts) } end end |
.setup?(language, resource = nil) ⇒ Boolean
Check if a language (or a specific resource for that language) is set up.
219 220 221 |
# File 'lib/kotoshu.rb', line 219 def self.setup?(language, resource = nil) ResourceManager.setup?(language, resource: resource) end |
.spellchecker ⇒ Spellchecker
Default spellchecker (singleton). Uses the configured default language. Cache-only — raises ResourceNotSetupError if the default language hasn't been set up via Kotoshu.setup.
125 126 127 128 129 130 131 132 |
# File 'lib/kotoshu.rb', line 125 def self.spellchecker return @spellchecker if @spellchecker lang = configuration.default_language raise ResourceNotSetupError.new(lang || "default", "spelling") if lang.nil? || lang.to_s.empty? @spellchecker = spellchecker_for(lang) end |
.spellchecker_for(language) ⇒ Spellchecker
Get a spellchecker for a specific language (cache-only, raises on miss).
143 144 145 146 147 148 149 150 |
# File 'lib/kotoshu.rb', line 143 def self.spellchecker_for(language) key = language.to_s @spellcheckers ||= {} @spellcheckers[key] ||= begin bundle = ResourceManager.resolve(language: language) Spellchecker.new(resource_bundle: bundle, config: configuration) end end |
.suggest(word, language: nil, **options) ⇒ Suggestions::SuggestionSet
Get spelling suggestions for a word. Hot path.
285 286 287 288 |
# File 'lib/kotoshu.rb', line 285 def self.suggest(word, language: nil, **) checker = language ? spellchecker_for(language) : spellchecker checker.suggest(word, **) end |
.suggestion_pipeline(*strategies) ⇒ Suggestions::Strategies::CompositeStrategy
Convenience method for creating a suggestion pipeline.
379 380 381 382 383 |
# File 'lib/kotoshu.rb', line 379 def self.suggestion_pipeline(*strategies) pipeline = Suggestions::Strategies::CompositeStrategy.new(name: :default) strategies.each { |s| pipeline.add(s) } pipeline end |
.supported_languages ⇒ Array<String>
Get all supported language codes.
470 471 472 |
# File 'lib/kotoshu.rb', line 470 def self.supported_languages Language.supported_codes end |
.trie(source = nil) ⇒ Core::Trie::Trie
Convenience method for creating a trie.
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 |
# File 'lib/kotoshu.rb', line 358 def self.trie(source = nil) case source when Array Core::Trie::Builder.from_array(source) when String if File.exist?(source) Core::Trie::Builder.from_file(source) else Core::Trie::Builder.from_string(source) end when nil Core::Trie::Trie.new else raise ArgumentError, "Invalid trie source: #{source.inspect}" end end |