Module: SemanticTextChunker

Defined in:
lib/semantic_text_chunker.rb,
lib/semantic_text_chunker/chunker.rb,
lib/semantic_text_chunker/version.rb,
lib/semantic_text_chunker/metadata.rb,
lib/semantic_text_chunker/chunk_builder.rb,
lib/semantic_text_chunker/embedders/base.rb,
lib/semantic_text_chunker/embedders/null.rb,
lib/semantic_text_chunker/embedders/cohere.rb,
lib/semantic_text_chunker/embedders/openai.rb,
lib/semantic_text_chunker/boundary_detector.rb,
lib/semantic_text_chunker/embedders/open_router.rb,
lib/semantic_text_chunker/splitters/sentence_splitter.rb

Defined Under Namespace

Modules: Embedders, Metadata, Splitters Classes: BoundaryDetector, ChunkBuilder, Chunker, EmbedderError

Constant Summary collapse

VERSION = 
"0.1.0"

Class Method Summary collapse

Class Method Details

.chunk(text, **opts) ⇒ Object 



11
12
13
 
# File 'lib/semantic_text_chunker.rb', line 11

def self.chunk(text, **opts)
  Chunker.new(**opts).chunk(text)
end

.chunk_with_metadata(text, **opts) ⇒ Object 



15
16
17
18
19
20
21
 
# File 'lib/semantic_text_chunker.rb', line 15

def self.chunk_with_metadata(text, **opts)
  metadata_keys = %i[title author chapter section source]
  chunker_opts  = opts.reject { |k, _| metadata_keys.include?(k) }
  metadata      = opts.select { |k, _| metadata_keys.include?(k) }

  Chunker.new(**chunker_opts).chunk_with_metadata(text, **metadata)
end