Class: Omnizip::Formats::XzImpl::BlockDecoder

Inherits:
Object
  • Object
show all
Defined in:
lib/omnizip/formats/xz_impl/block_decoder.rb

Overview

XZ Block decoder

Decodes a single XZ block which consists of:

  • Block Header
  • Compressed Data
  • Block Padding (to 4-byte boundary)
  • Check (CRC32/CRC64/SHA256)

Reference: /tmp/xz-source/src/liblzma/common/block_decoder.c

Defined Under Namespace

Classes: CountingInputStream

Constant Summary collapse

FILTER_LZMA2 =

Filter IDs

0x21
MAX_DICT_PROP =

XZ spec: max valid prop is 40 (gives ~2GB dict) Cap at 40 to prevent memory exhaustion from malformed files

40
MAX_DICT_SIZE =

64MB practical limit

64 * 1024 * 1024

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input, check_type) ⇒ BlockDecoder

Initialize block decoder

Parameters:

  • input (IO) —

    Input stream positioned at block header

  • check_type (Integer) —

    Check type (0=None, 1=CRC32, 4=CRC64, 10=SHA256)



88
89
90
91
92
93
94
95
 
# File 'lib/omnizip/formats/xz_impl/block_decoder.rb', line 88

def initialize(input, check_type)
  @input = input
  @check_type = check_type
  @new_input_after_block = nil # Track new input for stream decoder
  @data_already_decompressed = false # Track if LZMA2 already decoded the data
  @unpadded_size = nil # Track unpadded block size (for index validation)
  @uncompressed_size = nil # Track uncompressed size (for index validation)
end

Instance Attribute Details

#new_input_after_block ⇒ Object (readonly)

Accessor for new input after block (used by stream decoder for multi-block files)



47
48
49
 
# File 'lib/omnizip/formats/xz_impl/block_decoder.rb', line 47

def new_input_after_block
  @new_input_after_block
end

#uncompressed_size ⇒ Object (readonly)

Accessor for block size information (used for index validation)



49
50
51
 
# File 'lib/omnizip/formats/xz_impl/block_decoder.rb', line 49

def uncompressed_size
  @uncompressed_size
end

#unpadded_size ⇒ Object (readonly)

Accessor for block size information (used for index validation)



49
50
51
 
# File 'lib/omnizip/formats/xz_impl/block_decoder.rb', line 49

def unpadded_size
  @unpadded_size
end

Instance Method Details

#decode ⇒ Array<String, Hash>

Decode block

Returns:

  • (Array<String, Hash>) —

    Decompressed data and block info:

    • data: String (decompressed data)
    • info: Hash with header info

Raises:

  • (RuntimeError) —

    If block is invalid or checksum mismatch



103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
 
# File 'lib/omnizip/formats/xz_impl/block_decoder.rb', line 103

def decode
  # Parse block header
  header = BlockHeaderParser.parse(@input)

  # Read compressed data
  compressed_size = header[:compressed_size]
  check_size = Checksums::Verifier.check_size(@check_type)

  if ENV["XZ_BLOCK_DEBUG"]
    warn "DEBUG: decode - compressed_size=#{compressed_size.inspect}, check_type=#{@check_type}"
    warn "DEBUG: @input.class=#{@input.class}, @input.respond_to?(:pos)=#{@input.respond_to?(:pos)}"
    pos = @input.respond_to?(:pos) ? @input.pos : "N/A"
    warn "DEBUG: @input.pos=#{pos}"
  end

  if compressed_size.nil?
    # Compressed size is not present in header - need to determine block boundary
    # Read all remaining data
    all_remaining = @input.read

    # Decode LZMA2 and track how many bytes it consumes
    uncompressed_data, consumed_bytes = decode_lzma2_with_consumption_tracking(
      all_remaining: all_remaining,
      filters: header[:filters],
    )

    # Mark that data is already decompressed (LZMA2 only for now)
    @data_already_decompressed = true

    # Calculate padding and check positions
    # Block structure: [compressed data] [padding to 4-byte boundary] [check]
    padding_needed = (4 - (consumed_bytes % 4)) % 4
    check_start_pos = consumed_bytes + padding_needed

    # XZ Utils: Validate padding bytes are all zeros
    # Reference: /Users/mulgogi/src/external/xz/src/liblzma/common/block_decoder.c:131-139
    if padding_needed.positive?
      padding_bytes = all_remaining.byteslice(consumed_bytes,
                                              padding_needed)
      if padding_bytes.nil? || padding_bytes.bytesize < padding_needed
        raise Omnizip::FormatError,
              "Unexpected end of stream in block padding"
      end
      # Verify padding is all zeros
      unless padding_bytes.bytes.all?(0)
        raise Omnizip::FormatError,
              "Block padding contains non-zero bytes"
      end
    end

    if ENV["XZ_BLOCK_DEBUG"]
      warn "DEBUG: consumed_bytes=#{consumed_bytes}, padding_needed=#{padding_needed}, check_start_pos=#{check_start_pos}"
      warn "DEBUG: all_remaining.bytesize=#{all_remaining.bytesize}"
    end

    if check_start_pos + check_size > all_remaining.bytesize
      raise Omnizip::FormatError,
            "Invalid check position"
    end

    check_bytes = all_remaining.byteslice(check_start_pos, check_size)

    # Create new input with remaining data (after this block)
    total_block_size = check_start_pos + check_size
    data_after_block = all_remaining[total_block_size..]

    # Create new StringIO with remaining data
    new_input = StringIO.new(data_after_block)
    new_input.set_encoding(Encoding::BINARY)

    # Store the new input for the stream decoder to use
    @new_input_after_block = new_input
  else
    compressed_data = @input.read(compressed_size)
    if compressed_data.nil? || compressed_data.bytesize < compressed_size
      raise Omnizip::IOError,
            "Unexpected end of stream in compressed data: expected #{compressed_size} bytes"
    end

    # Read block padding (align to 4-byte boundary)
    # Block header is always 4-byte aligned, so we only need to pad the data
    padding_needed = (4 - (compressed_size % 4)) % 4
    if padding_needed.positive?
      padding = @input.read(padding_needed)
      if padding.nil? || padding.bytesize < padding_needed
        raise Omnizip::IOError,
              "Unexpected end of stream in block padding"
      end
      # Verify padding is all zeros
      unless padding.bytes.all?(0)
        raise Omnizip::FormatError,
              "Block padding contains non-zero bytes"
      end
    end

    # Read check
    if check_size.positive?
      check_bytes = @input.read(check_size)
      if check_bytes.nil? || check_bytes.bytesize < check_size
        raise Omnizip::IOError,
              "Unexpected end of stream in block check"
      end
    else
      check_bytes = ""
    end

    # When compressed_size is explicit, the input stream is now correctly
    # positioned at the start of the next block, so no need to create new input
  end

  # Decode filter chain (for now, just LZMA2)
  # Skip if data was already decompressed by decode_lzma2_with_consumption_tracking
  if @data_already_decompressed
    # LZMA2 was already decoded, but we may still have other filters to apply
    # For multi-filter chains, apply remaining filters in reverse order
    filters_to_process = header[:filters].dup
    # Remove the LZMA2 filter that was already processed
    filters_to_process.reject! { |f| f[:id] == FILTER_LZMA2 }

    if filters_to_process.empty?
      # No remaining filters
      uncompressed_data = @decompressed_data
    else
      # Apply remaining filters in reverse order
      data = @decompressed_data
      filters_to_process.reverse_each do |filter|
        data = decode_single_filter(data, filter)
      end
      uncompressed_data = data
    end
  else
    uncompressed_data = decode_filters(compressed_data,
                                       header[:filters])
  end

  # Verify uncompressed size matches header (if present)
  if header[:uncompressed_size] && (uncompressed_data.bytesize != header[:uncompressed_size])
    raise Omnizip::DecompressionError,
          "Uncompressed size mismatch: header says #{header[:uncompressed_size]}, got #{uncompressed_data.bytesize}"
  end

  # DEBUG: Show output before checksum check
  if ENV["DEBUG_CHECKSUM"]
    puts "DEBUG: uncompressed_data.bytesize=#{uncompressed_data.bytesize}"
    puts "DEBUG: first 100 bytes: #{uncompressed_data[0, 100].inspect}"
    puts "DEBUG: last 50 bytes: #{uncompressed_data[-50..].inspect}"
  end

  # Verify check
  unless Checksums::Verifier.verify(uncompressed_data, check_bytes,
                                    @check_type)
    raise Omnizip::ChecksumError,
          "Block checksum mismatch for check type #{@check_type}"
  end

  # Track block sizes for index validation (per XZ Utils index_hash.c)
  # Unpadded size = block header + compressed data + check (NO padding)
  # This is used to validate against the index records
  # Reference: xz-file-format-1.2.1.txt Section 3.3.2:
  #   "Unpadded Size is the size of the Block Header, Compressed Data,
  #    and Check fields. The Block Padding field is NOT included."
  @uncompressed_size = uncompressed_data.bytesize

  # Calculate unpadded block size (excludes padding per XZ spec)
  # Block structure: [block header] [compressed data] [padding] [check]
  # Reference: /Users/mulgogi/src/external/xz/src/liblzma/common/block_decoder.c
  header_size = header[:header_size] || 0
  if compressed_size.nil?
    # When compressed_size wasn't specified, we tracked consumed_bytes
    # unpadded_size = header_size + consumed_bytes + check_size (NO padding)
    # Note: BlockHeaderParser already consumed the header from input
    # For the size calculation, we need to include header size
    actual_compressed_size = consumed_bytes
    @unpadded_size = header_size + actual_compressed_size + check_size
  else
    # When compressed_size was specified
    @unpadded_size = header_size + compressed_size + check_size
  end

  uncompressed_data
end