Module: Omnizip::Chunked

Defined in:
lib/omnizip/chunked.rb,
lib/omnizip/chunked/reader.rb,
lib/omnizip/chunked/writer.rb,
lib/omnizip/chunked/memory_manager.rb

Overview

Chunked processing for memory-efficient large file handling

Defined Under Namespace

Classes: Configuration, MemoryError, MemoryManager, Reader, Writer

Class Method Summary collapse

Class Method Details

.compress_file(input, output, **options) ⇒ String

Compress file with chunked processing

Parameters:

  • input (String) —

    Input file path

  • output (String) —

    Output file path

  • options (Hash) —

    Compression options

Options Hash (**options):

  • :chunk_size (Integer) —

    Chunk size in bytes

  • :max_memory (Integer) —

    Maximum memory usage in bytes

  • :compression (Symbol) —

    Compression method

  • :progress (Proc) —

    Progress callback

Returns:

  • (String) —

    Output file path



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/omnizip/chunked.rb', line 44

def compress_file(input, output, **options)
  chunk_size = options[:chunk_size] || configuration.chunk_size
  options[:max_memory] || configuration.max_memory
  progress = options[:progress]

  unless File.exist?(input)
    raise Errno::ENOENT,
          "Input file not found: #{input}"
  end

  reader = Reader.new(input, chunk_size: chunk_size)
  total_size = reader.total_size
  processed = 0

  Omnizip::Zip::File.create(output) do |zip|
    basename = File.basename(input)

    # Use add with a block instead of private add_data
    zip.add(basename) do
      data = String.new(encoding: Encoding::BINARY)

      reader.each_chunk do |chunk|
        data << chunk
        processed += chunk.bytesize

        if progress
          percentage = (processed.to_f / total_size * 100).round(2)
          progress.call(processed, total_size, percentage)
        end
      end

      data
    end
  end

  output
end

.configuration ⇒ Object

Global configuration



25
26
27
 
# File 'lib/omnizip/chunked.rb', line 25

def configuration
  @configuration ||= Configuration.new
end

.configure {|config| ... } ⇒ Object

Configure chunked operations

Yields:

  • (config) —

    Configuration block



31
32
33
 
# File 'lib/omnizip/chunked.rb', line 31

def configure
  yield configuration
end

.decompress_file(input, output, **options) ⇒ String

Decompress file with chunked processing

Parameters:

  • input (String) —

    Input archive path

  • output (String) —

    Output file path

  • options (Hash) —

    Decompression options

Options Hash (**options):

  • :chunk_size (Integer) —

    Chunk size in bytes

  • :progress (Proc) —

    Progress callback

Returns:

  • (String) —

    Output file path



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/omnizip/chunked.rb', line 89

def decompress_file(input, output, **options)
  chunk_size = options[:chunk_size] || configuration.chunk_size
  progress = options[:progress]

  unless File.exist?(input)
    raise Errno::ENOENT,
          "Input archive not found: #{input}"
  end

  writer = Writer.new(output, chunk_size: chunk_size)
  processed = 0

  Omnizip::Zip::File.open(input) do |zip|
    entry = zip.entries.first
    total_size = entry.size

    # Read the full entry content
    content = zip.get_input_stream(entry)

    # Write in chunks
    offset = 0
    while offset < content.bytesize
      chunk = content.byteslice(offset, chunk_size) || ""
      break if chunk.empty?

      writer.write_chunk(chunk)
      processed += chunk.bytesize
      offset += chunk.bytesize

      if progress
        percentage = (processed.to_f / total_size * 100).round(2)
        progress.call(processed, total_size, percentage)
      end
    end
  end

  writer.close
  output
end