Module: Wp2txt::Bz2Validator

Defined in:
lib/wp2txt/bz2_validator.rb

Overview

Validates bz2 files for corruption and integrity Provides early detection of corrupt files before processing

Defined Under Namespace

Classes: ValidationResult

Constant Summary collapse

BZ2_MAGIC =

Bz2 magic bytes: "BZ" followed by version ('h') and block size ('1'-'9')

"BZ".freeze
BZ2_VERSION = 
"h".freeze
BZ2_BLOCK_SIZES = 
("1".."9").to_a.freeze
MIN_BZ2_SIZE =

Minimum valid bz2 file size (header + minimal compressed data)

14
TEST_CHUNK_SIZE =

Test chunk size for decompression validation

1_048_576

Class Method Summary collapse

Class Method Details

.file_info(path) ⇒ Hash

Get bz2 file information

Parameters:

  • path (String) —

    Path to bz2 file

Returns:

  • (Hash) —

    File information



222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
 
# File 'lib/wp2txt/bz2_validator.rb', line 222

def file_info(path)
  return nil unless File.exist?(path)

  header = File.binread(path, 4)
  {
    path: path,
    size: File.size(path),
    size_formatted: Wp2txt.format_file_size(File.size(path)),
    valid_header: header[0, 2] == BZ2_MAGIC,
    version: header[2],
    block_size: header[3]&.to_i,
    mtime: File.mtime(path)
  }
rescue IOError, Errno::ENOENT, Errno::EACCES
  nil
end

.find_bzip2_command ⇒ String?

Find available bzip2 decompression command

Returns:

  • (String, nil) —

    Path to command or nil



211
212
213
214
215
216
217
 
# File 'lib/wp2txt/bz2_validator.rb', line 211

def find_bzip2_command
  %w[lbzip2 pbzip2 bzip2 bzcat].each do |cmd|
    path = IO.popen(["which", cmd], err: File::NULL, &:read).strip
    return path unless path.empty?
  end
  nil
end

.test_decompression(path) ⇒ ValidationResult

Test decompression of first chunk

Parameters:

  • path (String) —

    Path to bz2 file

Returns:



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
 
# File 'lib/wp2txt/bz2_validator.rb', line 138

def test_decompression(path)
  bzcat_cmd = find_bzip2_command
  unless bzcat_cmd
    # Skip decompression test if no command available
    return ValidationResult.new(
      valid: true,
      error_type: nil,
      message: "Skipped decompression test (no bzip2 command)",
      details: { skipped: true }
    )
  end

  # Try to decompress first chunk
  begin
    # Use head to limit output and timeout to prevent hanging on large files
    output = nil
    error = nil

    IO.popen([bzcat_cmd, "-c", "-d", path], "rb", err: [:child, :out]) do |io|
      output = io.read(TEST_CHUNK_SIZE)
    end

    exit_status = $?.exitstatus

    if exit_status != 0 && (output.nil? || output.empty?)
      return ValidationResult.new(
        valid: false,
        error_type: :decompression_failed,
        message: "Decompression failed (corrupted data or truncated file)",
        details: { exit_status: exit_status }
      )
    end

    # Check if output looks like XML (Wikipedia dumps are XML)
    if output && output.bytesize > 0
      # Simple check for XML-like content
      sample = output[0, 1000].to_s.scrub("")
      unless sample.include?("<") && sample.include?(">")
        return ValidationResult.new(
          valid: false,
          error_type: :invalid_content,
          message: "Decompressed content does not appear to be XML",
          details: { sample_size: output.bytesize }
        )
      end
    end

    ValidationResult.new(
      valid: true,
      error_type: nil,
      message: "Decompression test passed",
      details: { bytes_tested: output&.bytesize || 0 }
    )
  rescue Errno::EPIPE
    # Broken pipe is OK - we only read partial output
    ValidationResult.new(
      valid: true,
      error_type: nil,
      message: "Decompression test passed (partial read)",
      details: {}
    )
  rescue IOError, Errno::ENOENT, Errno::EACCES => e
    ValidationResult.new(
      valid: false,
      error_type: :decompression_error,
      message: "Decompression error: #{e.message}",
      details: { error: e.class.name }
    )
  end
end

.validate(path, test_decompress: true) ⇒ ValidationResult

Perform full validation of a bz2 file

Parameters:

  • path (String) —

    Path to bz2 file

  • test_decompress (Boolean) (defaults to: true) —

    Whether to test decompression (slower but more thorough)

Returns:



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/wp2txt/bz2_validator.rb', line 37

def validate(path, test_decompress: true)
  # Check file exists
  unless File.exist?(path)
    return ValidationResult.new(
      valid: false,
      error_type: :not_found,
      message: "File not found",
      details: { path: path }
    )
  end

  # Check file size
  file_size = File.size(path)
  if file_size < MIN_BZ2_SIZE
    return ValidationResult.new(
      valid: false,
      error_type: :too_small,
      message: "File too small to be valid bz2 (#{file_size} bytes)",
      details: { size: file_size, minimum: MIN_BZ2_SIZE }
    )
  end

  # Check magic bytes
  magic_result = validate_magic_bytes(path)
  return magic_result unless magic_result.valid?

  # Test decompression if requested
  if test_decompress
    decompress_result = test_decompression(path)
    return decompress_result unless decompress_result.valid?
  end

  ValidationResult.new(
    valid: true,
    error_type: nil,
    message: "Valid bz2 file",
    details: { size: file_size, path: path }
  )
end

.validate_magic_bytes(path) ⇒ ValidationResult

Validate bz2 magic bytes

Parameters:

  • path (String) —

    Path to bz2 file

Returns:



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/wp2txt/bz2_validator.rb', line 87

def validate_magic_bytes(path)
  header = File.binread(path, 4)

  # Check "BZ" signature
  unless header[0, 2] == BZ2_MAGIC
    return ValidationResult.new(
      valid: false,
      error_type: :invalid_magic,
      message: "Invalid bz2 header (expected 'BZ', got '#{header[0, 2].inspect}')",
      details: { expected: BZ2_MAGIC, actual: header[0, 2] }
    )
  end

  # Check version byte ('h' for bzip2)
  unless header[2] == BZ2_VERSION
    return ValidationResult.new(
      valid: false,
      error_type: :invalid_version,
      message: "Invalid bz2 version byte (expected 'h', got '#{header[2].inspect}')",
      details: { expected: BZ2_VERSION, actual: header[2] }
    )
  end

  # Check block size byte ('1'-'9')
  unless BZ2_BLOCK_SIZES.include?(header[3])
    return ValidationResult.new(
      valid: false,
      error_type: :invalid_block_size,
      message: "Invalid bz2 block size (expected '1'-'9', got '#{header[3].inspect}')",
      details: { expected: BZ2_BLOCK_SIZES, actual: header[3] }
    )
  end

  ValidationResult.new(
    valid: true,
    error_type: nil,
    message: "Valid bz2 header",
    details: { version: header[2], block_size: header[3].to_i }
  )
rescue IOError, Errno::ENOENT, Errno::EACCES => e
  ValidationResult.new(
    valid: false,
    error_type: :read_error,
    message: "Cannot read file: #{e.message}",
    details: { error: e.class.name }
  )
end

.validate_quick(path) ⇒ ValidationResult

Quick validation (magic bytes only, no decompression test)

Parameters:

  • path (String) —

    Path to bz2 file

Returns:



80
81
82
 
# File 'lib/wp2txt/bz2_validator.rb', line 80

def validate_quick(path)
  validate(path, test_decompress: false)
end