Class: Ace::Bundle::Molecules::BundleChunker

Inherits:
Object
  • Object
show all
Defined in:
lib/ace/bundle/molecules/bundle_chunker.rb

Overview

BundleChunker splits large content into manageable chunks Configuration is loaded from Ace::Bundle.max_lines (ADR-022)

Performance Note: Current implementation loads full content into memory. For very large files (>100MB), consider implementing streaming chunking to reduce memory footprint. This optimization can be added in future versions.

Constant Summary collapse

DEFAULT_MAX_LINES =

Fallback value if config is not available

2_000
DEFAULT_CHUNK_SUFFIX = 
"_chunk"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(max_lines = nil) ⇒ BundleChunker

Returns a new instance of BundleChunker.

Parameters:

  • max_lines (Integer, nil) (defaults to: nil) —

    Override max lines per chunk (nil uses config)



22
23
24
 
# File 'lib/ace/bundle/molecules/bundle_chunker.rb', line 22

def initialize(max_lines = nil)
  @max_lines = max_lines || config_max_lines || DEFAULT_MAX_LINES
end

Instance Attribute Details

#max_lines ⇒ Object (readonly)

Returns the value of attribute max_lines.



19
20
21
 
# File 'lib/ace/bundle/molecules/bundle_chunker.rb', line 19

def max_lines
  @max_lines
end

Instance Method Details

#chunk_and_write(content, base_path, file_writer, options = {}) ⇒ Object

Split content and write all files (index + chunks)



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/ace/bundle/molecules/bundle_chunker.rb', line 75

def chunk_and_write(content, base_path, file_writer, options = {})
  chunk_result = chunk_content(content, base_path, options)

  unless chunk_result[:chunked]
    # Write single file
    return {
      chunked: false,
      files_written: 1,
      results: [
        file_writer.write(content, "#{base_path}.md", options)
      ]
    }
  end

  # Write index file and all chunks
  write_results = []

  # Write index file
  index_result = file_writer.write(
    chunk_result[:index_content],
    chunk_result[:index_file],
    options
  )
  write_results << index_result.merge(file_type: "index")

  # Write chunk files
  chunk_result[:chunk_files].each do |chunk_info|
    chunk_result_write = file_writer.write(
      chunk_info[:content],
      chunk_info[:path],
      options
    )
    write_results << chunk_result_write.merge(file_type: "chunk", chunk_number: chunk_info[:chunk_number])

    # Progress callback if provided
    if options[:progress_callback]
      options[:progress_callback].call("Wrote chunk #{chunk_info[:chunk_number]} of #{chunk_result[:total_chunks]}")
    end
  end

  {
    chunked: true,
    total_chunks: chunk_result[:total_chunks],
    files_written: write_results.size,
    results: write_results
  }
end

#chunk_content(content, base_path, options = {}) ⇒ Object

Split content into chunks



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/ace/bundle/molecules/bundle_chunker.rb', line 48

def chunk_content(content, base_path, options = {})
  opts = {
    chunk_suffix: DEFAULT_CHUNK_SUFFIX,
    include_metadata: true
  }.merge(options)

  return single_file_result(content, base_path) unless needs_chunking?(content)

  lines = content.lines
  chunks = split_into_chunks(lines)

  chunk_files = generate_chunk_files(chunks, base_path, opts)
  index_content = generate_index_content(chunk_files, base_path, opts)

  {
    chunked: true,
    total_chunks: chunks.size,
    max_lines: @max_lines,
    total_lines: lines.size,
    index_file: "#{base_path}.md",
    index_content: index_content,
    chunk_files: chunk_files,
    total_size: calculate_total_size(chunk_files)
  }
end

#needs_chunking?(content) ⇒ Boolean

Check if content needs chunking

Returns:

  • (Boolean) 


40
41
42
43
44
45
 
# File 'lib/ace/bundle/molecules/bundle_chunker.rb', line 40

def needs_chunking?(content)
  return false if content.nil? || content.empty?

  line_count = content.lines.size
  line_count > @max_lines
end