Class: Wp2txt::Runner

Inherits:

Object

• Object
• Wp2txt::Runner

Includes:

Wp2txt

Defined in:

lib/wp2txt.rb

Constant Summary

Constants included from Wp2txt

AUTHORITY_CONTROL_REGEX, AUTHORITY_CONTROL_TEMPLATES, BLANK_LINE_REGEX, BLOCK_EXTENSION_TAGS, BLOCK_MARKERS, BYTES_PER_GB, BYTES_PER_KB, BYTES_PER_MB, CATEGORY_LINE_REGEX, CATEGORY_NAMESPACES, CATEGORY_NAMESPACE_REGEX, CATEGORY_REGEX, CHRREF_TO_UTF_REGEX, CITATION_TEMPLATES, CITATION_TEMPLATE_REGEX, CLEANUP_EMPTY_PARENS_REGEX, CLEANUP_FILE_INCOMPLETE_REGEX, CLEANUP_FILE_LINE_REGEX, CLEANUP_KEY_VALUE_LINE_REGEX, CLEANUP_MIXED_WHITESPACE_REGEX, CLEANUP_MULTIPLE_PIPES_REGEX, CLEANUP_MULTIPLE_SPACES_REGEX, CLEANUP_MULTI_BLANK_REGEX, CLEANUP_ORPHANED_BRACKETS_REGEX, CLEANUP_ORPHANED_CLOSE_REGEX, CLEANUP_ORPHANED_OPEN_REGEX, CLEANUP_PIPE_CLOSE_REGEX, CLEANUP_PIPE_LINE_REGEX, CLEANUP_REGEX_01, CLEANUP_REGEX_02, CLEANUP_REGEX_03, CLEANUP_REGEX_04, CLEANUP_REGEX_05, CLEANUP_REGEX_06, CLEANUP_REGEX_07, CLEANUP_REGEX_08, CLEANUP_REMNANTS_REGEX, CLEANUP_REMNANTS_TEMPLATES, CLEANUP_STANDALONE_CLOSE_REGEX, CLEANUP_TRAILING_PIPE_REGEX, COMPLEX_REGEX_01, COMPLEX_REGEX_02, COMPLEX_REGEX_03, COMPLEX_REGEX_04, COMPLEX_REGEX_05, CONVERT_TEMPLATES, COUNTRY_CODE_REGEX, CURLY_SQUARE_BRACKET_REGEX, DEFAULTSORT_KEYWORDS, DEFAULT_BUFFER_SIZE, DEFAULT_CATEGORY_CACHE_EXPIRY_DAYS, DEFAULT_DUMP_EXPIRY_DAYS, DEFAULT_HTTP_TIMEOUT, DEFAULT_MARKERS, DEFAULT_PROGRESS_INTERVAL, DEFAULT_TOP_N_SECTIONS, DEF_MARKS_REGEX, DISPLAYTITLE_KEYWORDS, DOUBLE_CURLY_BRACKET_REGEX, DOUBLE_SQUARE_BRACKET_REGEX, DOUBLE_UNDERSCORE_MAGIC_REGEX, DOUBLE_UNDERSCORE_PATTERNS, ENTITIES, ESCAPE_NOWIKI_REGEX, EXTENSION_TAGS, EXTRA_ENTITIES, EXTRA_ENTITIES_REGEX, FILE_NAMESPACES, FILE_NAMESPACES_REGEX, FLAG_TEMPLATES, FLAG_TEMPLATE_REGEX, FORMATTING_TEMPLATES, FORMATTING_TEMPLATE_REGEX, FORMAT_REF_REGEX, HEADING_CODA_REGEX, HEADING_ONSET_REGEX, HTML_COMMENT_REGEX, HTML_DECODER, HTML_ENTITIES_PATH, HTML_HASH, HTML_REGEX, IMAGEMAP_COORD_REGEX, IMAGE_PARAMS, IMAGE_PARAMS_REGEX, IMAGE_PARAM_KEYS, INDEX_PROGRESS_THRESHOLD, INLINE_MARKERS, INTERWIKI_LINK_TEMPLATES, INTERWIKI_PREFIX_REGEX, IN_DEFINITION_REGEX, IN_HEADING_REGEX, IN_HTML_TABLE_REGEX, IN_HTML_TABLE_REGEX1, IN_HTML_TABLE_REGEX2, IN_INPUTBOX_REGEX, IN_INPUTBOX_REGEX1, IN_INPUTBOX_REGEX2, IN_LINK_REGEX, IN_MATH_REGEX, IN_MATH_REGEX1, IN_MATH_REGEX2, IN_ORDERED_REGEX, IN_PRE_REGEX, IN_SOURCE_REGEX, IN_SOURCE_REGEX1, IN_SOURCE_REGEX2, IN_TABLE_REGEX1, IN_TABLE_REGEX2, IN_UNORDERED_REGEX, ISOLATED_TAG_REGEX, ISOLATED_TEMPLATE_REGEX, LIST_MARKS_REGEX, LONE_ASTERISK_REGEX, MAGIC_WORD_LINE_REGEX, MAGIC_WORD_TEMPLATE_REGEX, MAKE_REFERENCE_REGEX_A, MAKE_REFERENCE_REGEX_B, MAKE_REFERENCE_REGEX_C, MAKE_REFERENCE_REGEX_D, MARKER_PATTERNS, MARKER_TYPES, MATH_ENTITIES, MATH_ENTITIES_REGEX, MAX_BUFFER_SIZE, MAX_HTTP_RETRIES, MAX_NESTING_ITERATIONS, MEDIAWIKI_DATA_PATH, MIN_BUFFER_SIZE, MIXED_SCRIPT_TEMPLATES, ML_LINK_END_REGEX, ML_LINK_ONSET_REGEX, ML_TEMPLATE_END_REGEX, ML_TEMPLATE_ONSET_REGEX, MNDASH_REGEX, NON_ARTICLE_NAMESPACES, NON_ARTICLE_NAMESPACE_REGEX, ONSET_BAR_REGEX, PRE_MARKS_REGEX, RACTOR_SHAREABLE_EXCLUDES, REDIRECT_KEYWORDS, REDIRECT_REGEX, REMOVE_DIRECTIVES_REGEX, REMOVE_DIRECTIVES_REGEX_GENERIC, REMOVE_EMPHASIS_REGEX, REMOVE_HR_REGEX, REMOVE_INLINE_REGEX, REMOVE_ISOLATED_REGEX, REMOVE_TAG_REGEX, REMOVE_TEMPLATES, REMOVE_TEMPLATES_REGEX, RESUME_METADATA_MAX_AGE_DAYS, RUBY_TEXT_TEMPLATES, SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, SELF_CLOSING_TAG_REGEX, SINGLE_CURLY_BRACKET_REGEX, SINGLE_SQUARE_BRACKET_REGEX, SISTER_PROJECTS, TEMPLATE_DATA_PATH, UNESCAPE_NOWIKI_REGEX, VERSION, WIKIMEDIA_PROJECT_NAMES, WIKIMEDIA_PROJECT_REGEX, WIKIPEDIA_ENTITIES_PATH

Instance Method Summary

• #extract_text(&block) ⇒ Object
• #fill_buffer ⇒ Object
• #get_newline ⇒ Object
• #get_page ⇒ Object
• #initialize(input_file, output_dir = ".", strip_tmarker = false, del_interfile = true) ⇒ Runner constructor

  A new instance of Runner.

• #prepare ⇒ Object

Methods included from Wp2txt

#apply_markers, #apply_pipe_trick, article_page?, #batch_file_mod, build_extra_entities_regex, build_template_regex, #chrref_to_utf, #cleanup, #collect_files, #convert_characters, #correct_inline_template, #correct_separator, days_to_seconds, #escape_nowiki, #extract_template_content, file_age_days, file_fresh?, #file_mod, #finalize_markers, #format_citation, format_file_size, #format_wiki, #format_wiki_regex_transform, #html_decoder, load_html_entities, load_mediawiki_data, load_template_data, make_constants_ractor_shareable!, #make_reference, #marker_placeholder, #mndash, #parse_markers_config, #process_external_links, #process_interwiki_links, #process_nested_single_pass, #process_nested_structure, #remove_complex, #remove_directive, #remove_emphasis, #remove_hr, #remove_html, #remove_inbetween, #remove_ref, #remove_table, #remove_tag, #remove_templates, #rename, #replace_paired_templates_with_marker, #replace_template_with_marker, #replace_wiki_table_with_marker, #sec_to_str, #special_chr, ssl_safe_get, #template_matches?, #unescape_nowiki

Constructor Details

#initialize(input_file, output_dir = ".", strip_tmarker = false, del_interfile = true) ⇒ Runner

Returns a new instance of Runner.

# File 'lib/wp2txt.rb', line 201

def initialize(input_file, output_dir = ".", strip_tmarker = false, del_interfile = true)
  @fp = nil
  @input_file = input_file
  @output_dir = output_dir
  @strip_tmarker = strip_tmarker
  @del_interfile = del_interfile
  prepare
end

Instance Method Details

#extract_text(&block) ⇒ Object

# File 'lib/wp2txt.rb', line 281

def extract_text(&block)
  title = nil
  output_text = +""
  pages = []
  data_empty = false

  until data_empty
    new_page = get_page
    if new_page
      pages << new_page
    else
      data_empty = true
    end
    next unless data_empty

    pages.each do |page|
      xmlns = '<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.5/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.5/ http://www.mediawiki.org/xml/export-0.5.xsd" version="0.5" xml:lang="en">' + "\n"
      xml = xmlns + page + "</mediawiki>"

      input = Nokogiri::XML(xml, nil, 'UTF-8')
      page = input.xpath("//xmlns:text").first
      pp_title = page.parent.parent.at_css "title"
      title = pp_title.content
      next if /:/ =~ title

      text = page.content
      text.gsub!(/<!--(.*?)-->/m) do |content|
        num_of_newlines = content.count("\n")
        if num_of_newlines.zero?
          +""
        else
          "\n" * num_of_newlines
        end
      end
      article = Article.new(text, title, @strip_tmarker)
      page_text = block.call(article)
      output_text << page_text
    end

    output_text = cleanup(output_text)
    unless output_text.empty?
      outfilename = File.join(@output_dir, @outfile_base + ".txt")
      @fp = File.open(outfilename, "w")
      @fp.puts(output_text)
      @fp.close
    end
    @file_pointer.close
    File.delete(@input_file) if @del_interfile
    output_text = +""
  end
end

#fill_buffer ⇒ Object

# File 'lib/wp2txt.rb', line 219

def fill_buffer
  loop do
    begin
      new_lines = @file_pointer.read(10_485_760)
    rescue IOError, Errno::EIO, Errno::ENOENT, Errno::EPIPE
      return nil
    end
    return nil unless new_lines

    # temp_buf is filled with text split by "\n"
    temp_buf = []
    ss = StringScanner.new(new_lines)
    temp_buf << ss[0] while ss.scan(/.*?\n/m)
    temp_buf << ss.rest unless ss.eos?

    new_first_line = temp_buf.shift
    @buffer.last << new_first_line
    # Use end_with? instead of [-1, 1] for clarity and performance
    @buffer << +"" if new_first_line.end_with?("\n")
    @buffer.concat(temp_buf) unless temp_buf.empty?
    @buffer << +"" if @buffer.last.end_with?("\n")
    break if @buffer.size > 1
  end
  true
end

#get_newline ⇒ Object

# File 'lib/wp2txt.rb', line 245

def get_newline
  @buffer ||= [+""]
  if @buffer.size == 1 && !fill_buffer
    nil
  elsif @buffer.empty?
    nil
  else
    @buffer.shift
  end
end

#get_page ⇒ Object

# File 'lib/wp2txt.rb', line 256

def get_page
  inside_page = false
  page = +""
  while (line = get_newline)
    case line
    when /<page>/
      page << line
      inside_page = true
      next
    when %r{</page>}
      page << line
      inside_page = false
      break
    end
    page << line if inside_page
  end
  if page.empty?
    false
  else
    page.force_encoding("utf-8")
  end
rescue ::Encoding::InvalidByteSequenceError, ::Encoding::UndefinedConversionError
  page
end

#prepare ⇒ Object

# File 'lib/wp2txt.rb', line 210

def prepare
  @infile_size = File.stat(@input_file).size
  file = File.open(@input_file, "r:UTF-8")
  @file_pointer = file
  @outfile_base = File.basename(@input_file, ".*")
  @total_size = 0
  true
end