Class: Wp2txt::Splitter

Inherits:
Object
  • Object
show all
Includes:
Wp2txt
Defined in:
lib/wp2txt.rb

Constant Summary

Constants included from Wp2txt

AUTHORITY_CONTROL_REGEX, AUTHORITY_CONTROL_TEMPLATES, BLANK_LINE_REGEX, BLOCK_EXTENSION_TAGS, BLOCK_MARKERS, BYTES_PER_GB, BYTES_PER_KB, BYTES_PER_MB, CATEGORY_LINE_REGEX, CATEGORY_NAMESPACES, CATEGORY_NAMESPACE_REGEX, CATEGORY_REGEX, CHRREF_TO_UTF_REGEX, CITATION_TEMPLATES, CITATION_TEMPLATE_REGEX, CLEANUP_EMPTY_PARENS_REGEX, CLEANUP_FILE_INCOMPLETE_REGEX, CLEANUP_FILE_LINE_REGEX, CLEANUP_KEY_VALUE_LINE_REGEX, CLEANUP_MIXED_WHITESPACE_REGEX, CLEANUP_MULTIPLE_PIPES_REGEX, CLEANUP_MULTIPLE_SPACES_REGEX, CLEANUP_MULTI_BLANK_REGEX, CLEANUP_ORPHANED_BRACKETS_REGEX, CLEANUP_ORPHANED_CLOSE_REGEX, CLEANUP_ORPHANED_OPEN_REGEX, CLEANUP_PIPE_CLOSE_REGEX, CLEANUP_PIPE_LINE_REGEX, CLEANUP_REGEX_01, CLEANUP_REGEX_02, CLEANUP_REGEX_03, CLEANUP_REGEX_04, CLEANUP_REGEX_05, CLEANUP_REGEX_06, CLEANUP_REGEX_07, CLEANUP_REGEX_08, CLEANUP_REMNANTS_REGEX, CLEANUP_REMNANTS_TEMPLATES, CLEANUP_STANDALONE_CLOSE_REGEX, CLEANUP_TRAILING_PIPE_REGEX, COMPLEX_REGEX_01, COMPLEX_REGEX_02, COMPLEX_REGEX_03, COMPLEX_REGEX_04, COMPLEX_REGEX_05, CONVERT_TEMPLATES, COUNTRY_CODE_REGEX, CURLY_SQUARE_BRACKET_REGEX, DEFAULTSORT_KEYWORDS, DEFAULT_BUFFER_SIZE, DEFAULT_CATEGORY_CACHE_EXPIRY_DAYS, DEFAULT_DUMP_EXPIRY_DAYS, DEFAULT_HTTP_TIMEOUT, DEFAULT_MARKERS, DEFAULT_PROGRESS_INTERVAL, DEFAULT_TOP_N_SECTIONS, DEF_MARKS_REGEX, DISPLAYTITLE_KEYWORDS, DOUBLE_CURLY_BRACKET_REGEX, DOUBLE_SQUARE_BRACKET_REGEX, DOUBLE_UNDERSCORE_MAGIC_REGEX, DOUBLE_UNDERSCORE_PATTERNS, ENTITIES, ESCAPE_NOWIKI_REGEX, EXTENSION_TAGS, EXTRA_ENTITIES, EXTRA_ENTITIES_REGEX, FILE_NAMESPACES, FILE_NAMESPACES_REGEX, FLAG_TEMPLATES, FLAG_TEMPLATE_REGEX, FORMATTING_TEMPLATES, FORMATTING_TEMPLATE_REGEX, FORMAT_REF_REGEX, HEADING_CODA_REGEX, HEADING_ONSET_REGEX, HTML_COMMENT_REGEX, HTML_DECODER, HTML_ENTITIES_PATH, HTML_HASH, HTML_REGEX, IMAGEMAP_COORD_REGEX, IMAGE_PARAMS, IMAGE_PARAMS_REGEX, IMAGE_PARAM_KEYS, INDEX_PROGRESS_THRESHOLD, INLINE_MARKERS, INTERWIKI_LINK_TEMPLATES, INTERWIKI_PREFIX_REGEX, IN_DEFINITION_REGEX, IN_HEADING_REGEX, IN_HTML_TABLE_REGEX, IN_HTML_TABLE_REGEX1, IN_HTML_TABLE_REGEX2, IN_INPUTBOX_REGEX, IN_INPUTBOX_REGEX1, IN_INPUTBOX_REGEX2, IN_LINK_REGEX, IN_MATH_REGEX, IN_MATH_REGEX1, IN_MATH_REGEX2, IN_ORDERED_REGEX, IN_PRE_REGEX, IN_SOURCE_REGEX, IN_SOURCE_REGEX1, IN_SOURCE_REGEX2, IN_TABLE_REGEX1, IN_TABLE_REGEX2, IN_UNORDERED_REGEX, ISOLATED_TAG_REGEX, ISOLATED_TEMPLATE_REGEX, LIST_MARKS_REGEX, LONE_ASTERISK_REGEX, MAGIC_WORD_LINE_REGEX, MAGIC_WORD_TEMPLATE_REGEX, MAKE_REFERENCE_REGEX_A, MAKE_REFERENCE_REGEX_B, MAKE_REFERENCE_REGEX_C, MAKE_REFERENCE_REGEX_D, MARKER_PATTERNS, MARKER_TYPES, MATH_ENTITIES, MATH_ENTITIES_REGEX, MAX_BUFFER_SIZE, MAX_HTTP_RETRIES, MAX_NESTING_ITERATIONS, MEDIAWIKI_DATA_PATH, MIN_BUFFER_SIZE, MIXED_SCRIPT_TEMPLATES, ML_LINK_END_REGEX, ML_LINK_ONSET_REGEX, ML_TEMPLATE_END_REGEX, ML_TEMPLATE_ONSET_REGEX, MNDASH_REGEX, NON_ARTICLE_NAMESPACES, NON_ARTICLE_NAMESPACE_REGEX, ONSET_BAR_REGEX, PRE_MARKS_REGEX, RACTOR_SHAREABLE_EXCLUDES, REDIRECT_KEYWORDS, REDIRECT_REGEX, REMOVE_DIRECTIVES_REGEX, REMOVE_DIRECTIVES_REGEX_GENERIC, REMOVE_EMPHASIS_REGEX, REMOVE_HR_REGEX, REMOVE_INLINE_REGEX, REMOVE_ISOLATED_REGEX, REMOVE_TAG_REGEX, REMOVE_TEMPLATES, REMOVE_TEMPLATES_REGEX, RESUME_METADATA_MAX_AGE_DAYS, RUBY_TEXT_TEMPLATES, SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, SELF_CLOSING_TAG_REGEX, SINGLE_CURLY_BRACKET_REGEX, SINGLE_SQUARE_BRACKET_REGEX, SISTER_PROJECTS, TEMPLATE_DATA_PATH, UNESCAPE_NOWIKI_REGEX, VERSION, WIKIMEDIA_PROJECT_NAMES, WIKIMEDIA_PROJECT_REGEX, WIKIPEDIA_ENTITIES_PATH

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Wp2txt

#apply_markers, #apply_pipe_trick, article_page?, #batch_file_mod, build_extra_entities_regex, build_template_regex, #chrref_to_utf, #cleanup, #collect_files, #convert_characters, #correct_inline_template, #correct_separator, days_to_seconds, #escape_nowiki, #extract_template_content, file_age_days, file_fresh?, #file_mod, #finalize_markers, #format_citation, format_file_size, #format_wiki, #format_wiki_regex_transform, #html_decoder, load_html_entities, load_mediawiki_data, load_template_data, make_constants_ractor_shareable!, #make_reference, #marker_placeholder, #mndash, #parse_markers_config, #process_external_links, #process_interwiki_links, #process_nested_single_pass, #process_nested_structure, #remove_complex, #remove_directive, #remove_emphasis, #remove_hr, #remove_html, #remove_inbetween, #remove_ref, #remove_table, #remove_tag, #remove_templates, #rename, #replace_paired_templates_with_marker, #replace_template_with_marker, #replace_wiki_table_with_marker, #sec_to_str, #special_chr, ssl_safe_get, #template_matches?, #unescape_nowiki

Constructor Details

#initialize(input_file, output_dir = ".", tfile_size = 10, bz2_gem = false, &progress_callback) ⇒ Splitter

Returns a new instance of Splitter.



15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/wp2txt.rb', line 15

def initialize(input_file, output_dir = ".", tfile_size = 10, bz2_gem = false, &progress_callback)
  @fp = nil
  @input_file = input_file
  @output_dir = output_dir
  @tfile_size = tfile_size
  require "bzip2-ruby" if bz2_gem
  @bz2_gem = bz2_gem
  @progress_callback = progress_callback
  @last_progress_time = Time.now
  prepare
end

Instance Attribute Details

#file_index ⇒ Object (readonly)

Returns the value of attribute file_index.



13
14
15
 
# File 'lib/wp2txt.rb', line 13

def file_index
  @file_index
end

#size_read ⇒ Object (readonly)

Returns the value of attribute size_read.



13
14
15
 
# File 'lib/wp2txt.rb', line 13

def size_read
  @size_read
end

Instance Method Details

#command_exist?(command) ⇒ Boolean

check if a given command exists: return the path if it does, return false if not

Returns:

  • (Boolean) 


54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/wp2txt.rb', line 54

def command_exist?(command)
  basename = File.basename(command)
  print "Checking #{basename}: "
  begin
    # Use IO.popen instead of open("| ...") for Ruby 4.0 compatibility
    path = IO.popen(["which", command], err: File::NULL, &:read).strip
    if path.empty?
      path = IO.popen(["which", basename], err: File::NULL, &:read).strip
    end

    if path.empty?
      puts "#{basename} not found"
      false
    else
      puts "detected [#{path}]"
      path
    end
  rescue Errno::ENOENT, Errno::EPIPE, IOError
    puts "#{basename} not found"
    false
  end
end

#file_size(file) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/wp2txt.rb', line 27

def file_size(file)
  size = 0
  unit = 10_485_760
  star = 0
  before = Time.now.to_f

  loop do
    begin
      a = file.read(unit)
    rescue IOError, Errno::EIO, Errno::ENOENT
      a = nil
    end
    break unless a

    present = Time.now.to_f
    size += a.size

    next if present - before <= 0.3

    star = 0 if star > 10
    star += 1
    before = present
  end
  size
end

#fill_buffer ⇒ Object

read text data from bz2 compressed file by 1 megabyte



109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/wp2txt.rb', line 109

def fill_buffer
  loop do
    begin
      new_lines = @file_pointer.read(10_485_760)
    rescue IOError, Errno::EIO, Errno::ENOENT, Errno::EPIPE
      return nil
    end
    return nil unless new_lines

    # temp_buf is filled with text split by "\n"
    temp_buf = []
    ss = StringScanner.new(new_lines)
    temp_buf << ss[0] while ss.scan(/.*?\n/m)
    temp_buf << ss.rest unless ss.eos?

    new_first_line = temp_buf.shift
    @buffer.last << new_first_line
    # Use end_with? instead of [-1, 1] for clarity and performance
    @buffer << +"" if new_first_line.end_with?("\n")
    @buffer.concat(temp_buf) unless temp_buf.empty?
    @buffer << +"" if @buffer.last.end_with?("\n")
    break if @buffer.size > 1
  end
  true
end

#get_newline ⇒ Object 



135
136
137
138
139
140
141
142
143
144
 
# File 'lib/wp2txt.rb', line 135

def get_newline
  @buffer ||= [+""]
  if @buffer.size == 1 && !fill_buffer
    nil
  elsif @buffer.empty?
    nil
  else
    @buffer.shift
  end
end

#prepare ⇒ Object

check the size of input file (bz2 or plain xml) when decompressed



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/wp2txt.rb', line 78

def prepare
  # if output_dir is not specified, output in the same directory
  # as the imput file
  @output_dir = File.dirname(@input_file) if !@output_dir && @input_file

  if /.bz2$/ =~ @input_file
    if @bz2_gem
      file = Bzip2::Reader.new File.open(@input_file, "r:UTF-8")
    elsif Gem.win_platform?
      file = IO.popen(["bunzip2.exe", "-c", @input_file])
    elsif (bzpath = command_exist?("lbzip2") || command_exist?("pbzip2") || command_exist?("bzip2"))
      file = IO.popen([bzpath, "-c", "-d", @input_file])
    end
  else # meaning that it is a text file
    @infile_size = File.stat(@input_file).size
    file = File.open(@input_file, "r:UTF-8")
  end

  # create basename of output file
  @outfile_base = File.basename(@input_file, ".*") + "-"
  @total_size = 0
  @file_index = 1
  outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
  @outfiles = []
  @outfiles << outfilename
  @fp = File.open(outfilename, "w")
  @file_pointer = file
  true
end

#split_file ⇒ Object 



146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/wp2txt.rb', line 146

def split_file
  output_text = +""
  end_flag = false
  while (text = get_newline)
    @count ||= 0
    @count += 1
    @size_read ||= 0
    @size_read += text.bytesize
    @total_size += text.bytesize
    output_text << text
    end_flag = true if @total_size > (@tfile_size * 1024 * 1024)

    # Report progress every 5 seconds
    report_progress

    # never close the file until the end of the page even if end_flag is on
    next unless end_flag && %r{</page} =~ text

    @fp.puts(output_text)
    output_text = +""
    @total_size = 0
    end_flag = false
    @fp.close
    @file_index += 1
    outfilename = File.join(@output_dir, @outfile_base + @file_index.to_s)
    @outfiles << outfilename
    @fp = File.open(outfilename, "w")
  end
  @fp.puts(output_text) unless output_text.empty?
  @fp.close

  if outfilename && File.size(outfilename).zero?
    File.delete(outfilename)
    @outfiles.delete(outfilename)
  end

  rename(@outfiles, "xml")
end