Class: Omnizip::Formats::SevenZip::Parser

Inherits:
Object
  • Object
show all
Includes:
Constants
Defined in:
lib/omnizip/formats/seven_zip/parser.rb

Overview

Binary data parser for .7z format Implements variable-length encoding and bit vector handling

Constant Summary

Constants included from Constants

Constants::MAJOR_VERSION, Constants::MAX_NUM_BONDS, Constants::MAX_NUM_CODERS, Constants::MAX_NUM_PACK_STREAMS, Constants::MINOR_VERSION, Constants::SIGNATURE, Constants::SIGNATURE_SIZE, Constants::START_HEADER_SIZE

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(data) ⇒ Parser

Initialize parser with binary data

Parameters:

  • data (String) —

    Binary data to parse



16
17
18
19
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 16

def initialize(data)
  @data = data.b
  @position = 0
end

Instance Attribute Details

#data ⇒ Object (readonly)

Returns the value of attribute data.



11
12
13
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 11

def data
  @data
end

#position ⇒ Object (readonly)

Returns the value of attribute position.



11
12
13
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 11

def position
  @position
end

Instance Method Details

#eof? ⇒ Boolean

Check if at end of data

Returns:

  • (Boolean) —

    true if no more data



185
186
187
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 185

def eof?
  @position >= @data.bytesize
end

#expect_property(expected) ⇒ Object

Expect specific property ID

Parameters:

  • expected (Integer) —

    Expected property ID

Raises:

  • (RuntimeError) —

    if property doesn't match



610
611
612
613
614
615
616
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 610

def expect_property(expected)
  actual = read_byte
  return if actual == expected

  raise "Expected property 0x#{expected.to_s(16)}, " \
        "got 0x#{actual.to_s(16)}"
end

#peek_byte ⇒ Integer

Peek at next byte without advancing position

Returns:

  • (Integer) —

    Next byte value

Raises:

  • (EOFError) 


176
177
178
179
180
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 176

def peek_byte
  raise EOFError if @position >= @data.bytesize

  @data.getbyte(@position)
end

#read_anti(entries) ⇒ Object

Read anti flags

Parameters:



561
562
563
564
565
566
567
568
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 561

def read_anti(entries)
  skip_size
  anti_files = entries.select { |e| !e.has_stream && !e.is_empty }
  anti_bits = read_raw_bit_vector(anti_files.size)
  anti_files.each_with_index do |entry, i|
    entry.is_anti = anti_bits[i]
  end
end

#read_attributes(entries) ⇒ Object

Read file attributes

Parameters:



594
595
596
597
598
599
600
601
602
603
604
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 594

def read_attributes(entries)
  skip_size
  defined_bits = read_bit_vector(entries.size)
  external = read_byte

  return unless external.zero?

  entries.each_with_index do |entry, i|
    entry.attributes = read_uint32 if defined_bits[i]
  end
end

#read_bit_vector(num_items) ⇒ Object

Bit vector with an all-defined marker byte (kCRC/kWinAttrib defined-bits layout)



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 137

def read_bit_vector(num_items)
  all_defined = read_byte
  num_bytes = (num_items + 7) / 8

  if all_defined.zero?
    # Explicit bit vector
    raise EOFError if @position + num_bytes > @data.bytesize

    bits_data = @data[@position, num_bytes]
    @position += num_bytes
    decode_bit_vector(bits_data, num_items)
  else
    # All bits are set (return 1 for each item, not true)
    Array.new(num_items, 1)
  end
end

#read_byte ⇒ Integer

Read a single byte

Returns:

  • (Integer) —

    Byte value (0-255)

Raises:

  • (EOFError) —

    if no more data available



25
26
27
28
29
30
31
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 25

def read_byte
  raise EOFError, "End of data" if @position >= @data.bytesize

  byte = @data.getbyte(@position)
  @position += 1
  byte
end

#read_bytes(count) ⇒ String

Read raw bytes

Parameters:

  • count (Integer) —

    Number of bytes to read

Returns:

  • (String) —

    Binary string

Raises:

  • (EOFError) 


158
159
160
161
162
163
164
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 158

def read_bytes(count)
  raise EOFError if @position + count > @data.bytesize

  bytes = @data[@position, count]
  @position += count
  bytes
end

#read_coder(coder) ⇒ Object

Read coder definition

Parameters:



315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 315

def read_coder(coder)
  main_byte = read_byte

  # Extract coder flags
  id_size = main_byte & 0x0F
  has_attributes = main_byte.anybits?(0x20)
  complex_streams = main_byte.anybits?(0x10)

  # Read method ID
  method_id = 0
  id_size.times do
    method_id = (method_id << 8) | read_byte
  end
  coder.method_id = method_id

  # Read stream counts if complex
  if complex_streams
    coder.num_in_streams = read_number
    coder.num_out_streams = read_number
  else
    coder.num_in_streams = 1
    coder.num_out_streams = 1
  end

  # Read properties if present
  return unless has_attributes

  props_size = read_number
  coder.properties = read_bytes(props_size)
end

#read_empty_file(entries) ⇒ Object

Read empty file flags

Parameters:



547
548
549
550
551
552
553
554
555
556
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 547

def read_empty_file(entries)
  skip_size
  empty_files = entries.reject(&:has_stream)
  empty_bits = read_raw_bit_vector(empty_files.size)
  empty_files.each_with_index do |entry, i|
    # Bit set = zero-byte FILE (not a directory)
    entry.is_empty = empty_bits[i] == 1
    entry.is_dir = false if entry.is_empty
  end
end

#read_empty_stream(entries) ⇒ Object

Read empty stream flags

Parameters:



533
534
535
536
537
538
539
540
541
542
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 533

def read_empty_stream(entries)
  skip_size
  # kEmptyStream carries a RAW bit vector — no all-defined
  # marker byte (unlike kCRC/kWinAttrib)
  empty_stream = read_raw_bit_vector(entries.size)
  entries.each_with_index do |entry, i|
    entry.has_stream = empty_stream[i].zero?
    entry.is_dir = (empty_stream[i] == 1)
  end
end

#read_files_info ⇒ Array<Models::FileEntry>

Read files info section Contains file metadata (names, timestamps, attributes)

Returns:



464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 464

def read_files_info
  num_files = read_number
  entries = Array.new(num_files) { Models::FileEntry.new }

  # Read file properties
  until eof? || peek_byte == PropertyId::K_END
    prop_type = read_byte

    case prop_type
    when PropertyId::NAME
      read_names(entries)
    when PropertyId::EMPTY_STREAM
      read_empty_stream(entries)
    when PropertyId::EMPTY_FILE
      read_empty_file(entries)
    when PropertyId::ANTI
      read_anti(entries)
    when PropertyId::CTIME
      read_timestamps(entries, :ctime)
    when PropertyId::ATIME
      read_timestamps(entries, :atime)
    when PropertyId::MTIME
      read_timestamps(entries, :mtime)
    when PropertyId::WIN_ATTRIB
      read_attributes(entries)
    when PropertyId::DUMMY
      skip_data
    else
      skip_data
    end
  end

  read_byte if !eof? && peek_byte == PropertyId::K_END

  entries
end

#read_folder(folder) ⇒ Object

Read single folder definition

Parameters:



269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 269

def read_folder(folder)
  num_coders = read_number
  raise "Too many coders" if num_coders > Constants::MAX_NUM_CODERS

  num_in_streams = 0
  num_out_streams = 0

  # Read coders
  num_coders.times do
    coder = Models::CoderInfo.new
    read_coder(coder)
    folder.coders << coder
    num_in_streams += coder.num_in_streams
    num_out_streams += coder.num_out_streams
  end

  # Read bind pairs
  num_bind_pairs = num_out_streams - 1
  num_bind_pairs.times do
    in_index = read_number
    out_index = read_number
    folder.bind_pairs << [in_index, out_index]
  end

  # Read pack stream indices
  num_pack_streams = num_in_streams - num_bind_pairs
  if num_pack_streams == 1
    # Single pack stream - find unused input
    (0...num_in_streams).each do |i|
      used = folder.bind_pairs.any? { |pair| pair[0] == i }
      unless used
        folder.pack_stream_indices << i
        break
      end
    end
  else
    # Multiple pack streams - read indices
    num_pack_streams.times do
      folder.pack_stream_indices << read_number
    end
  end
end

#read_folders(stream_info) ⇒ Object

Read folders section Contains compression method and coder information

Parameters:

  • stream_info (StreamInfo) —

    Stream info to populate



246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 246

def read_folders(stream_info)
  num_folders = read_number

  # Read External flag (similar to names)
  external = read_byte

  # Only read folders if they're stored inline (external == 0)
  if external.zero?
    # Read each folder
    num_folders.times do
      folder = Models::Folder.new
      read_folder(folder)
      stream_info.folders << folder
    end
  else
    # External folders not supported
    raise "External folders not supported"
  end
end

#read_id ⇒ Integer

Read property ID

Returns:

  • (Integer) —

    Property ID



93
94
95
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 93

def read_id
  read_number
end

#read_names(entries) ⇒ Object

Read file names

Parameters:



504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 504

def read_names(entries)
  # Size of all names in bytes
  size = read_number
  start_pos = @position
  external = read_byte

  if external.zero?
    # Names stored inline as UTF-16LE null-terminated strings
    entries.each do |entry|
      name_bytes = +""
      loop do
        ch1 = read_byte
        ch2 = read_byte
        break if ch1.zero? && ch2.zero?

        name_bytes << [ch1, ch2].pack("CC")
      end
      entry.name = name_bytes.encode("UTF-8", "UTF-16LE")
    end
  end

  # Ensure we consumed expected bytes
  consumed = @position - start_pos
  skip(size - consumed) if consumed < size
end

#read_number ⇒ Integer

Read variable-length integer (7-Zip VLI format)

7-Zip VLI encoding uses the first byte's high bits to determine the number of additional bytes:

0xxxxxxx               : value = xxxxxxx (0-127)
10xxxxxx BYTE y[1]     : value = (xxxxxx << 8) + y
110xxxxx BYTE y[2]     : value = (xxxxx << 16) + y
1110xxxx BYTE y[3]     : value = (xxxx << 24) + y
...up to 8 bytes total

Returns:

  • (Integer) —

    Decoded number



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 44

def read_number
  first_byte = read_byte

  # Single byte encoding (0-127)
  return first_byte if first_byte.nobits?(0x80)

  # 7-Zip VLI: extra bytes are in little-endian order.
  # Per 7-Zip SDK CInByte2::ReadNumber():
  #   - Each set bit in first_byte (from MSB) means one more extra byte
  #   - Extra bytes are placed at increasing byte positions (LE)
  #   - The remaining data bits from first_byte go at the highest position
  mask = 0x80
  value = 0
  shift = 0

  while first_byte.anybits?(mask)
    value |= (read_byte << shift)
    shift += 8
    mask >>= 1
  end

  # The data bits from first_byte go at the highest position
  # When mask reaches 0 (first_byte == 0xFF, 8 extra bytes), there are no data bits
  data_bits = mask.zero? ? 0 : (first_byte & (mask - 1))
  value |= (data_bits << shift)

  value
end

#read_number32 ⇒ Integer

Read 32-bit variable-length number

Returns:

  • (Integer) —

    Number value (max 32-bit)

Raises:

  • (RuntimeError) —

    if value exceeds 32-bit range



77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 77

def read_number32
  first_byte = peek_byte
  if first_byte.nobits?(0x80)
    @position += 1
    return first_byte
  end

  value = read_number
  raise "Unsupported 32-bit value" if value >= 0x80000000

  value
end

#read_pack_info(stream_info) ⇒ Object

Read pack info section Contains information about packed streams

Parameters:

  • stream_info (StreamInfo) —

    Stream info to populate



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 200

def read_pack_info(stream_info)
  # Read pack position
  stream_info.pack_pos = read_number

  # Read number of pack streams
  num_pack_streams = read_number

  # Read properties in any order (7z format allows variable ordering)
  sizes_read = false

  until eof? || peek_byte == PropertyId::K_END
    prop_type = peek_byte

    case prop_type
    when PropertyId::SIZE
      read_byte
      num_pack_streams.times do
        stream_info.pack_sizes << read_number
      end
      sizes_read = true
    when PropertyId::CRC
      read_byte
      defined_vec = read_bit_vector(num_pack_streams)
      num_pack_streams.times do |i|
        stream_info.pack_crcs << (defined_vec[i] ? read_uint32 : nil)
      end
    when PropertyId::K_END
      break
    else
      # Unknown property - skip it
      read_byte
      skip_data unless eof?
    end
  end

  # Verify required SIZE property was present
  raise "Missing SIZE property in pack info" unless sizes_read

  # K_END is optional in some 7-Zip versions
  read_byte if !eof? && peek_byte == PropertyId::K_END
end

#read_raw_bit_vector(num_items) ⇒ Array<Boolean>

Read bit vector Format: 1 byte flag, then either all 1s or bit array

Raw MSB-first bit vector without an all-defined marker (kEmptyStream/kEmptyFile/kAnti layout)

Parameters:

  • num_items (Integer) —

    Number of items in bit vector

Returns:

  • (Array<Boolean>) —

    Bit vector

Raises:

  • (EOFError) 


126
127
128
129
130
131
132
133
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 126

def read_raw_bit_vector(num_items)
  num_bytes = (num_items + 7) / 8
  raise EOFError if @position + num_bytes > @data.bytesize

  bits_data = @data[@position, num_bytes]
  @position += num_bytes
  decode_bit_vector(bits_data, num_items)
end

#read_substreams_info(stream_info) ⇒ Object

Read substreams info section Maps files to compressed streams

Parameters:

  • stream_info (StreamInfo) —

    Stream info to populate



400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 400

def read_substreams_info(stream_info)
  # Read number of unpack streams per folder
  if !eof? && peek_byte == PropertyId::NUM_UNPACK_STREAM
    read_byte
    stream_info.folders.each do
      stream_info.num_unpack_streams_in_folders << read_number
    end
  else
    # Default: one stream per folder
    stream_info.folders.size.times do
      stream_info.num_unpack_streams_in_folders << 1
    end
  end

  # Read unpack sizes
  if !eof? && peek_byte == PropertyId::SIZE
    read_byte
    stream_info.folders.each_with_index do |folder, i|
      num_streams = stream_info.num_unpack_streams_in_folders[i]
      start_idx = stream_info.unpack_sizes.size

      if num_streams > 1
        (num_streams - 1).times do
          size = read_number
          stream_info.unpack_sizes << size
        end
      end
      # Last stream size = folder's final output size - sum of sizes we just read for this folder
      folder_total_size = folder.uncompressed_size
      sum = stream_info.unpack_sizes[start_idx..]&.sum || 0
      last_size = folder_total_size - sum
      stream_info.unpack_sizes << last_size
    end
  else
    # No SIZE property - use folder unpack sizes directly
    # This happens when each folder has exactly one stream
    stream_info.folders.each_with_index do |folder, i|
      num_streams = stream_info.num_unpack_streams_in_folders[i]
      if num_streams == 1
        # Single stream - use folder's total unpack size
        folder_size = folder.unpack_sizes.empty? ? 0 : folder.unpack_sizes.sum
        stream_info.unpack_sizes << folder_size
      end
    end
  end

  # Read digests (CRCs)
  num_digests = stream_info.num_unpack_streams_in_folders.sum
  if !(eof? || peek_byte == PropertyId::K_END) && (peek_byte == PropertyId::CRC)
    read_byte
    defined_vec = read_bit_vector(num_digests)
    num_digests.times do |i|
      stream_info.digests << (defined_vec[i] ? read_uint32 : nil)
    end
  end

  # K_END is optional in some 7-Zip versions
  read_byte if !eof? && peek_byte == PropertyId::K_END
end

#read_timestamps(entries, attr) ⇒ Object

Read timestamps

Parameters:

  • entries (Array<Models::FileEntry>) —

    File entries

  • attr (Symbol) —

    Attribute name (:mtime, :atime, :ctime)



574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 574

def read_timestamps(entries, attr)
  skip_size
  defined_bits = read_bit_vector(entries.size)
  external = read_byte

  return unless external.zero?

  entries.each_with_index do |entry, i|
    next unless defined_bits[i]

    time_val = read_uint64
    # Convert Windows FILETIME to Ruby Time
    # (100-nanosecond intervals since 1601-01-01)
    entry.public_send(:"#{attr}=", windows_time_to_unix(time_val))
  end
end

#read_uint32 ⇒ Integer

Read fixed-size unsigned 32-bit integer (little-endian)

Returns:

  • (Integer) —

    32-bit value

Raises:

  • (EOFError) 


100
101
102
103
104
105
106
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 100

def read_uint32
  raise EOFError if @position + 4 > @data.bytesize

  value = @data[@position, 4].unpack1("V")
  @position += 4
  value
end

#read_uint64 ⇒ Integer

Read fixed-size unsigned 64-bit integer (little-endian)

Returns:

  • (Integer) —

    64-bit value

Raises:

  • (EOFError) 


111
112
113
114
115
116
117
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 111

def read_uint64
  raise EOFError if @position + 8 > @data.bytesize

  value = @data[@position, 8].unpack1("Q<")
  @position += 8
  value
end

#read_unpack_info(stream_info) ⇒ Object

Read unpack info section Contains information about unpacked streams

Parameters:

  • stream_info (StreamInfo) —

    Stream info to populate



350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 350

def read_unpack_info(stream_info)
  # Read properties in any order (7z format allows variable ordering)
  folders_read = false
  unpack_sizes_read = false

  until eof? || peek_byte == PropertyId::K_END
    prop_type = read_byte

    case prop_type
    when PropertyId::FOLDER
      read_folders(stream_info)
      folders_read = true
    when PropertyId::CODERS_UNPACK_SIZE
      # Read unpack sizes for each folder based on numOutStreams
      stream_info.folders.each do |folder|
        # For folders with 0 coders (Copy method), treat as having 1 output stream
        num_out_streams = folder.num_out_streams
        num_out_streams = 1 if num_out_streams.zero?

        num_out_streams.times do
          folder.unpack_sizes << read_number
        end
      end
      unpack_sizes_read = true
    when PropertyId::CRC
      # Optional: read CRCs
      defined_vec = read_bit_vector(stream_info.num_folders)
      stream_info.num_folders.times do |i|
        stream_info.folders[i].unpack_crc = read_uint32 if defined_vec[i]
      end
    when PropertyId::K_END
      break
    else
      # Skip unknown properties
      skip_data unless eof?
    end
  end

  # Verify required properties were present
  raise "Missing FOLDER property in unpack info" unless folders_read
  raise "Missing CODERS_UNPACK_SIZE property in unpack info" unless unpack_sizes_read

  # K_END is optional in some 7-Zip versions
  read_byte if !eof? && peek_byte == PropertyId::K_END
end

#remaining ⇒ Integer

Get remaining byte count

Returns:

  • (Integer) —

    Bytes remaining



192
193
194
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 192

def remaining
  @data.bytesize - @position
end

#skip(count) ⇒ Object

Skip bytes

Parameters:

  • count (Integer) —

    Number of bytes to skip



169
170
171
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 169

def skip(count)
  @position += count
end

#skip_data ⇒ Object

Skip property data



624
625
626
627
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 624

def skip_data
  size = read_number
  skip(size)
end

#skip_size ⇒ Object

Skip size field



619
620
621
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 619

def skip_size
  read_number
end

#windows_time_to_unix(windows_time) ⇒ Time

Convert Windows FILETIME to Unix timestamp

Parameters:

  • windows_time (Integer) —

    Windows FILETIME

Returns:

  • (Time) —

    Ruby Time object



633
634
635
636
637
638
639
640
641
 
# File 'lib/omnizip/formats/seven_zip/parser.rb', line 633

def windows_time_to_unix(windows_time)
  # Windows FILETIME epoch: 1601-01-01
  # Unix epoch: 1970-01-01
  # Difference: 11644473600 seconds
  unix_time = (windows_time / 10_000_000) - 11_644_473_600
  Time.at(unix_time)
rescue StandardError
  nil
end