Module: Legion::Extensions::Knowledge::Helpers::Chunker

Extended by:
Logging::Helper, Settings::Helper
Defined in:
lib/legion/extensions/knowledge/helpers/chunker.rb

Constant Summary collapse

CHARS_PER_TOKEN = 
4

Class Method Summary collapse

Class Method Details

.chunk(sections:, max_tokens: nil, overlap_tokens: nil) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/legion/extensions/knowledge/helpers/chunker.rb', line 17

def chunk(sections:, max_tokens: nil, overlap_tokens: nil)
  resolved_max     = max_tokens     || settings[:chunker][:max_tokens]
  resolved_overlap = overlap_tokens || settings[:chunker][:overlap_tokens]

  max_chars     = resolved_max * CHARS_PER_TOKEN
  overlap_chars = resolved_overlap * CHARS_PER_TOKEN

  chunks = []
  sections.each do |section|
    chunks.concat(split_section(section, max_chars, overlap_chars))
  end
  chunks
end