Class: RubyLlmMesh::Rag::Chunker

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm_mesh/rag/chunker.rb

Overview

Lightweight text chunker with optional overlap for RAG pipelines.

Constant Summary collapse

DEFAULT_SIZE = 
800
DEFAULT_OVERLAP = 
100

Instance Method Summary collapse

Constructor Details

#initialize(size: DEFAULT_SIZE, overlap: DEFAULT_OVERLAP, separator: /\n{2,}|\n|\s+/) ⇒ Chunker

Returns a new instance of Chunker.

Raises:

  • (ArgumentError) 


10
11
12
13
14
15
16
 
# File 'lib/ruby_llm_mesh/rag/chunker.rb', line 10

def initialize(size: DEFAULT_SIZE, overlap: DEFAULT_OVERLAP, separator: /\n{2,}|\n|\s+/)
  raise ArgumentError, "overlap must be less than size" if overlap >= size

  @size = size
  @overlap = overlap
  @separator = separator
end

Instance Method Details

#chunk(text) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/ruby_llm_mesh/rag/chunker.rb', line 18

def chunk(text)
  return [] if text.nil? || text.strip.empty?

  paragraphs = text.to_s.split(@separator).map(&:strip).reject(&:empty?)
  chunks = []
  buffer = +""

  paragraphs.each do |piece|
    candidate = buffer.empty? ? piece : "#{buffer} #{piece}"
    if candidate.length <= @size
      buffer = candidate
    else
      chunks << buffer unless buffer.empty?
      buffer = overlap_tail(buffer)
      buffer = buffer.empty? ? piece : "#{buffer} #{piece}"
      while buffer.length > @size
        chunks << buffer[0, @size]
        buffer = overlap_tail(buffer[0, @size]) + buffer[@size..]
      end
    end
  end

  chunks << buffer unless buffer.empty?
  chunks
end