Class: Wp2txt::StreamProcessor

Inherits:
Object
  • Object
show all
Includes:
Wp2txt
Defined in:
lib/wp2txt/stream_processor.rb

Overview

StreamProcessor handles streaming decompression and XML parsing without creating intermediate files

Constant Summary collapse

MIN_BUFFER_SIZE =

Buffer size bounds from constants

Wp2txt::MIN_BUFFER_SIZE
MAX_BUFFER_SIZE = 
Wp2txt::MAX_BUFFER_SIZE
DEFAULT_BUFFER_SIZE = 
Wp2txt::DEFAULT_BUFFER_SIZE

Constants included from Wp2txt

AUTHORITY_CONTROL_REGEX, AUTHORITY_CONTROL_TEMPLATES, BLANK_LINE_REGEX, BLOCK_EXTENSION_TAGS, BLOCK_MARKERS, BYTES_PER_GB, BYTES_PER_KB, BYTES_PER_MB, CATEGORY_LINE_REGEX, CATEGORY_NAMESPACES, CATEGORY_NAMESPACE_REGEX, CATEGORY_REGEX, CHRREF_TO_UTF_REGEX, CITATION_TEMPLATES, CITATION_TEMPLATE_REGEX, CLEANUP_EMPTY_PARENS_REGEX, CLEANUP_FILE_INCOMPLETE_REGEX, CLEANUP_FILE_LINE_REGEX, CLEANUP_KEY_VALUE_LINE_REGEX, CLEANUP_MIXED_WHITESPACE_REGEX, CLEANUP_MULTIPLE_PIPES_REGEX, CLEANUP_MULTIPLE_SPACES_REGEX, CLEANUP_MULTI_BLANK_REGEX, CLEANUP_ORPHANED_BRACKETS_REGEX, CLEANUP_ORPHANED_CLOSE_REGEX, CLEANUP_ORPHANED_OPEN_REGEX, CLEANUP_PIPE_CLOSE_REGEX, CLEANUP_PIPE_LINE_REGEX, CLEANUP_REGEX_01, CLEANUP_REGEX_02, CLEANUP_REGEX_03, CLEANUP_REGEX_04, CLEANUP_REGEX_05, CLEANUP_REGEX_06, CLEANUP_REGEX_07, CLEANUP_REGEX_08, CLEANUP_REMNANTS_REGEX, CLEANUP_REMNANTS_TEMPLATES, CLEANUP_STANDALONE_CLOSE_REGEX, CLEANUP_TRAILING_PIPE_REGEX, COMPLEX_REGEX_01, COMPLEX_REGEX_02, COMPLEX_REGEX_03, COMPLEX_REGEX_04, COMPLEX_REGEX_05, CONVERT_TEMPLATES, COUNTRY_CODE_REGEX, CURLY_SQUARE_BRACKET_REGEX, DEFAULTSORT_KEYWORDS, DEFAULT_CATEGORY_CACHE_EXPIRY_DAYS, DEFAULT_DUMP_EXPIRY_DAYS, DEFAULT_HTTP_TIMEOUT, DEFAULT_MARKERS, DEFAULT_PROGRESS_INTERVAL, DEFAULT_TOP_N_SECTIONS, DEF_MARKS_REGEX, DISPLAYTITLE_KEYWORDS, DOUBLE_CURLY_BRACKET_REGEX, DOUBLE_SQUARE_BRACKET_REGEX, DOUBLE_UNDERSCORE_MAGIC_REGEX, DOUBLE_UNDERSCORE_PATTERNS, ENTITIES, ESCAPE_NOWIKI_REGEX, EXTENSION_TAGS, EXTRA_ENTITIES, EXTRA_ENTITIES_REGEX, FILE_NAMESPACES, FILE_NAMESPACES_REGEX, FLAG_TEMPLATES, FLAG_TEMPLATE_REGEX, FORMATTING_TEMPLATES, FORMATTING_TEMPLATE_REGEX, FORMAT_REF_REGEX, HEADING_CODA_REGEX, HEADING_ONSET_REGEX, HTML_COMMENT_REGEX, HTML_DECODER, HTML_ENTITIES_PATH, HTML_HASH, HTML_REGEX, IMAGEMAP_COORD_REGEX, IMAGE_PARAMS, IMAGE_PARAMS_REGEX, IMAGE_PARAM_KEYS, INDEX_PROGRESS_THRESHOLD, INLINE_MARKERS, INTERWIKI_LINK_TEMPLATES, INTERWIKI_PREFIX_REGEX, IN_DEFINITION_REGEX, IN_HEADING_REGEX, IN_HTML_TABLE_REGEX, IN_HTML_TABLE_REGEX1, IN_HTML_TABLE_REGEX2, IN_INPUTBOX_REGEX, IN_INPUTBOX_REGEX1, IN_INPUTBOX_REGEX2, IN_LINK_REGEX, IN_MATH_REGEX, IN_MATH_REGEX1, IN_MATH_REGEX2, IN_ORDERED_REGEX, IN_PRE_REGEX, IN_SOURCE_REGEX, IN_SOURCE_REGEX1, IN_SOURCE_REGEX2, IN_TABLE_REGEX1, IN_TABLE_REGEX2, IN_UNORDERED_REGEX, ISOLATED_TAG_REGEX, ISOLATED_TEMPLATE_REGEX, LIST_MARKS_REGEX, LONE_ASTERISK_REGEX, MAGIC_WORD_LINE_REGEX, MAGIC_WORD_TEMPLATE_REGEX, MAKE_REFERENCE_REGEX_A, MAKE_REFERENCE_REGEX_B, MAKE_REFERENCE_REGEX_C, MAKE_REFERENCE_REGEX_D, MARKER_PATTERNS, MARKER_TYPES, MATH_ENTITIES, MATH_ENTITIES_REGEX, MAX_HTTP_RETRIES, MAX_NESTING_ITERATIONS, MEDIAWIKI_DATA_PATH, MIXED_SCRIPT_TEMPLATES, ML_LINK_END_REGEX, ML_LINK_ONSET_REGEX, ML_TEMPLATE_END_REGEX, ML_TEMPLATE_ONSET_REGEX, MNDASH_REGEX, NON_ARTICLE_NAMESPACES, NON_ARTICLE_NAMESPACE_REGEX, ONSET_BAR_REGEX, PRE_MARKS_REGEX, RACTOR_SHAREABLE_EXCLUDES, REDIRECT_KEYWORDS, REDIRECT_REGEX, REMOVE_DIRECTIVES_REGEX, REMOVE_DIRECTIVES_REGEX_GENERIC, REMOVE_EMPHASIS_REGEX, REMOVE_HR_REGEX, REMOVE_INLINE_REGEX, REMOVE_ISOLATED_REGEX, REMOVE_TAG_REGEX, REMOVE_TEMPLATES, REMOVE_TEMPLATES_REGEX, RESUME_METADATA_MAX_AGE_DAYS, RUBY_TEXT_TEMPLATES, SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, SELF_CLOSING_TAG_REGEX, SINGLE_CURLY_BRACKET_REGEX, SINGLE_SQUARE_BRACKET_REGEX, SISTER_PROJECTS, TEMPLATE_DATA_PATH, UNESCAPE_NOWIKI_REGEX, VERSION, WIKIMEDIA_PROJECT_NAMES, WIKIMEDIA_PROJECT_REGEX, WIKIPEDIA_ENTITIES_PATH

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Wp2txt

#apply_markers, #apply_pipe_trick, article_page?, #batch_file_mod, build_extra_entities_regex, build_template_regex, #chrref_to_utf, #cleanup, #collect_files, #convert_characters, #correct_inline_template, #correct_separator, days_to_seconds, #escape_nowiki, #extract_template_content, file_age_days, file_fresh?, #file_mod, #finalize_markers, #format_citation, format_file_size, #format_wiki, #format_wiki_regex_transform, #html_decoder, load_html_entities, load_mediawiki_data, load_template_data, make_constants_ractor_shareable!, #make_reference, #marker_placeholder, #mndash, #parse_markers_config, #process_external_links, #process_interwiki_links, #process_nested_single_pass, #process_nested_structure, #remove_complex, #remove_directive, #remove_emphasis, #remove_hr, #remove_html, #remove_inbetween, #remove_ref, #remove_table, #remove_tag, #remove_templates, #rename, #replace_paired_templates_with_marker, #replace_template_with_marker, #replace_wiki_table_with_marker, #sec_to_str, #special_chr, ssl_safe_get, #template_matches?, #unescape_nowiki

Constructor Details

#initialize(input_path, bz2_gem: false, adaptive_buffer: true, validate_bz2: true, skip_redirects: true) ⇒ StreamProcessor

Returns a new instance of StreamProcessor.



22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/wp2txt/stream_processor.rb', line 22

def initialize(input_path, bz2_gem: false, adaptive_buffer: true, validate_bz2: true, skip_redirects: true)
  @input_path = input_path
  @bz2_gem = bz2_gem
  @buffer = +""
  @file_pointer = nil
  @adaptive_buffer = adaptive_buffer
  @buffer_size = adaptive_buffer ? calculate_optimal_buffer_size : DEFAULT_BUFFER_SIZE
  @pages_processed = 0
  @bytes_read = 0
  @validate_bz2 = validate_bz2
  @skip_redirects = skip_redirects
  @redirects_skipped = 0
end

Instance Attribute Details

#buffer_size ⇒ Object (readonly)

Returns the value of attribute buffer_size.



20
21
22
 
# File 'lib/wp2txt/stream_processor.rb', line 20

def buffer_size
  @buffer_size
end

#bytes_read ⇒ Object (readonly)

Returns the value of attribute bytes_read.



20
21
22
 
# File 'lib/wp2txt/stream_processor.rb', line 20

def bytes_read
  @bytes_read
end

#pages_processed ⇒ Object (readonly)

Returns the value of attribute pages_processed.



20
21
22
 
# File 'lib/wp2txt/stream_processor.rb', line 20

def pages_processed
  @pages_processed
end

#redirects_skipped ⇒ Object (readonly)

Returns the value of attribute redirects_skipped.



20
21
22
 
# File 'lib/wp2txt/stream_processor.rb', line 20

def redirects_skipped
  @redirects_skipped
end

Instance Method Details

#calculate_optimal_buffer_size ⇒ Object

Calculate optimal buffer size based on available memory



53
54
55
56
57
 
# File 'lib/wp2txt/stream_processor.rb', line 53

def calculate_optimal_buffer_size
  MemoryMonitor.optimal_buffer_size
rescue StandardError
  DEFAULT_BUFFER_SIZE
end

#each_page ⇒ Object

Iterate over each page in the input Yields [title, text] for each page



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/wp2txt/stream_processor.rb', line 66

def each_page
  return enum_for(:each_page) unless block_given?

  if File.directory?(@input_path)
    # Process XML files in directory
    Dir.glob(File.join(@input_path, "*.xml")).sort.each do |xml_file|
      process_xml_file(xml_file) { |title, text| yield title, text }
    end
  elsif @input_path.end_with?(".bz2")
    # Process bz2 compressed file with streaming
    process_bz2_stream { |title, text| yield title, text }
  elsif @input_path.end_with?(".xml")
    # Process single XML file
    process_xml_file(@input_path) { |title, text| yield title, text }
  else
    raise ArgumentError, "Unsupported input format: #{@input_path}"
  end
end

#memory_stats ⇒ Object

Get current memory statistics



60
61
62
 
# File 'lib/wp2txt/stream_processor.rb', line 60

def memory_stats
  MemoryMonitor.memory_stats
end

#stats ⇒ Object

Get processing statistics (public API for monitoring)



86
87
88
89
90
91
92
93
94
95
 
# File 'lib/wp2txt/stream_processor.rb', line 86

def stats
  {
    pages_processed: @pages_processed,
    redirects_skipped: @redirects_skipped,
    bytes_read: @bytes_read,
    buffer_size: @buffer_size,
    current_buffer_length: @buffer.bytesize,
    memory: memory_stats
  }
end

#validate_input(quick: false, raise_on_error: false) ⇒ Bz2Validator::ValidationResult

Validate bz2 file before processing

Parameters:

  • quick (Boolean) (defaults to: false) —

    Use quick validation (header only) vs full validation

Returns:

Raises:



40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/wp2txt/stream_processor.rb', line 40

def validate_input(quick: false, raise_on_error: false)
  return nil unless @input_path.end_with?(".bz2")

  result = quick ? Bz2Validator.validate_quick(@input_path) : Bz2Validator.validate(@input_path)

  if !result.valid? && raise_on_error
    raise Wp2txt::FileIOError, "Invalid bz2 file: #{result.message}"
  end

  result
end