Module: Wp2txt

Included in:
Article, Runner, Splitter, StreamProcessor
Defined in:
lib/wp2txt.rb,
lib/wp2txt/cli.rb,
lib/wp2txt/regex.rb,
lib/wp2txt/utils.rb,
lib/wp2txt/cli_ui.rb,
lib/wp2txt/config.rb,
lib/wp2txt/article.rb,
lib/wp2txt/version.rb,
lib/wp2txt/constants.rb,
lib/wp2txt/extractor.rb,
lib/wp2txt/formatter.rb,
lib/wp2txt/file_utils.rb,
lib/wp2txt/index_cache.rb,
lib/wp2txt/magic_words.rb,
lib/wp2txt/multistream.rb,
lib/wp2txt/bz2_validator.rb,
lib/wp2txt/output_writer.rb,
lib/wp2txt/ractor_worker.rb,
lib/wp2txt/category_cache.rb,
lib/wp2txt/memory_monitor.rb,
lib/wp2txt/text_processing.rb,
lib/wp2txt/parser_functions.rb,
lib/wp2txt/stream_processor.rb,
lib/wp2txt/global_data_cache.rb,
lib/wp2txt/section_extractor.rb,
lib/wp2txt/template_expander.rb

Defined Under Namespace

Modules: Bz2Validator, CLI, CliUI, Extractor, Formatter, MemoryMonitor, RactorWorker Classes: Article, CacheError, CategoryCache, CategoryFetcher, Config, DumpManager, EncodingError, Error, FileIOError, GlobalDataCache, IndexCache, MagicWordExpander, MultistreamIndex, MultistreamReader, NetworkError, NullProgressBar, NullSpinner, OutputWriter, ParseError, ParserFunctions, Runner, SectionExtractor, SectionStatsCollector, Splitter, StreamProcessor, TemplateExpander

Constant Summary collapse

MEDIAWIKI_DATA_PATH =

Data file paths

File.join(__dir__, "data", "mediawiki_aliases.json")
HTML_ENTITIES_PATH = 
File.join(__dir__, "data", "html_entities.json")
WIKIPEDIA_ENTITIES_PATH = 
File.join(__dir__, "data", "wikipedia_entities.json")
TEMPLATE_DATA_PATH = 
File.join(__dir__, "data", "template_aliases.json")
HTML_DECODER =

variables to save resource for generating regexps those with a trailing number 1 represent opening tag/markup those with a trailing number 2 represent closing tag/markup those without a trailing number contain both opening/closing tags/markups

HTMLEntities.new
ENTITIES = 
['&nbsp;', '&lt;', '&gt;', '&amp;', '&quot;'].zip([' ', '<', '>', '&', '"'])
HTML_HASH = 

      
        
HTML_REGEX =
          
        

        
Regexp.new("(" + HTML_HASH.keys.join("|") + ")")

      
        
EXTRA_ENTITIES =
          

  

    
Extra HTML entities loaded from JSON files (WHATWG + Wikipedia-specific)
These supplement the HTMLEntities gem which only covers ~250 basic entities
Data sources:


- lib/wp2txt/data/html_entities.json (2000+ WHATWG entities)
- lib/wp2txt/data/wikipedia_entities.json (Wikipedia-specific)


  





  



        

        
load_html_entities.freeze

      
        
EXTRA_ENTITIES_REGEX =
          
        

        
build_extra_entities_regex

      
        
MATH_ENTITIES =
          

  

    
Legacy alias for backward compatibility


  





  



        

        
EXTRA_ENTITIES

      
        
MATH_ENTITIES_REGEX =
          
        

        
EXTRA_ENTITIES_REGEX

      
        
ML_TEMPLATE_ONSET_REGEX =
          
        

        
Regexp.new('^\{\{[^\}]*$')

      
        
ML_TEMPLATE_END_REGEX =
          
        

        
Regexp.new('\}\}\s*$')

      
        
        
Regexp.new('^\[\[(?!.*\]\]\s*$).*$')

      
        
        
Regexp.new('\]\]\s*$')

      
        
ISOLATED_TEMPLATE_REGEX =
          
        

        
Regexp.new('^\s*\{\{.+\}\}\s*$')

      
        
ISOLATED_TAG_REGEX =
          
        

        
Regexp.new('^\s*\<[^\<\>]+\>.+\<[^\<\>]+\>\s*$')

      
        
        
Regexp.new('^\s*\[.*\]\s*$')

      
        
IN_INPUTBOX_REGEX =
          
        

        
Regexp.new('<inputbox>.*?<\/inputbox>')

      
        
IN_INPUTBOX_REGEX1 =
          
        

        
Regexp.new('<inputbox>')

      
        
IN_INPUTBOX_REGEX2 =
          
        

        
Regexp.new('<\/inputbox>')

      
        
IN_SOURCE_REGEX =
          
        

        
Regexp.new('<source.*?>.*?<\/source>')

      
        
IN_SOURCE_REGEX1 =
          
        

        
Regexp.new('<source.*?>')

      
        
IN_SOURCE_REGEX2 =
          
        

        
Regexp.new('<\/source>')

      
        
IN_MATH_REGEX =
          
        

        
Regexp.new('<math.*?>.*?<\/math>')

      
        
IN_MATH_REGEX1 =
          
        

        
Regexp.new('<math.*?>')

      
        
IN_MATH_REGEX2 =
          
        

        
Regexp.new('<\/math>')

      
        
IN_HEADING_REGEX =
          
        

        
Regexp.new('^=+.*?=+\s*$')

      
        
IN_HTML_TABLE_REGEX =
          
        

        
Regexp.new("<table.*?><\/table>")

      
        
IN_HTML_TABLE_REGEX1 =
          
        

        
Regexp.new('<table\b')

      
        
IN_HTML_TABLE_REGEX2 =
          
        

        
Regexp.new('<\/\s*table>')

      
        
HTML_COMMENT_REGEX =
          

  

    
HTML comments (can span multiple lines)


  





  



        

        
Regexp.new('<!--.*?-->', Regexp::MULTILINE)

      
        
IN_TABLE_REGEX1 =
          
        

        
Regexp.new('^\s*\{\|')

      
        
IN_TABLE_REGEX2 =
          
        

        
Regexp.new('^\|\}.*?$')

      
        
IN_UNORDERED_REGEX =
          
        

        
Regexp.new('^\*')

      
        
IN_ORDERED_REGEX =
          
        

        
Regexp.new('^\#')

      
        
IN_PRE_REGEX =
          
        

        
Regexp.new('^ ')

      
        
IN_DEFINITION_REGEX =
          
        

        
Regexp.new('^[\;\:]')

      
        
BLANK_LINE_REGEX =
          
        

        
Regexp.new('^\s*$')

      
        
REDIRECT_KEYWORDS =
          

  

    
Multilingual redirect keyword support
Data source: MediaWiki API (siteinfo) via scripts/fetch_mediawiki_data.rb


  





  



        

        
load_mediawiki_data.dig("magic_words", "redirect")&.join("|") || "REDIRECT"

      
        
REDIRECT_REGEX =
          
        

        
Regexp.new('[#＃](?:' + REDIRECT_KEYWORDS + ')\s*:?\s*\[\[([^\]]+)\]\]', Regexp::IGNORECASE)

      
        
REMOVE_TAG_REGEX =
          
        

        
Regexp.new("\<[^\<\>]*\>")

      
        
REMOVE_DIRECTIVES_REGEX_GENERIC =
          

  

    
Legacy generic pattern for double-underscore directives
Note: Data-driven REMOVE_DIRECTIVES_REGEX is defined later (after DOUBLE_UNDERSCORE_MAGIC_REGEX)


  





  



        

        
Regexp.new("\_\_[^\_]*\_\_")

      
        
REMOVE_EMPHASIS_REGEX =
          
        

        
Regexp.new('(' + Regexp.escape("''") + '+)(.+?)\1')

      
        
CHRREF_TO_UTF_REGEX =
          
        

        
Regexp.new('&#(x?)([0-9a-fA-F]+);')

      
        
MNDASH_REGEX =
          
        

        
Regexp.new('\{(mdash|ndash|–)\}')

      
        
REMOVE_HR_REGEX =
          
        

        
Regexp.new('^\s*\-{4,}\s*$')

      
        
MAKE_REFERENCE_REGEX_A =
          
        

        
Regexp.new('<br ?\/>', Regexp::IGNORECASE)

      
        
MAKE_REFERENCE_REGEX_B =
          
        

        
Regexp.new('<ref[^>]*\/>', Regexp::IGNORECASE)

      
        
MAKE_REFERENCE_REGEX_C =
          
        

        
Regexp.new('<ref[^>]*>', Regexp::IGNORECASE)

      
        
MAKE_REFERENCE_REGEX_D =
          
        

        
Regexp.new('<\/ref>', Regexp::IGNORECASE)

      
        
FORMAT_REF_REGEX =
          
        

        
Regexp.new('\[ref\](.*?)\[\/ref\]', Regexp::MULTILINE)

      
        
HEADING_ONSET_REGEX =
          
        

        
Regexp.new('^(\=+)\s+')

      
        
HEADING_CODA_REGEX =
          
        

        
Regexp.new('\s+(\=+)$')

      
        
LIST_MARKS_REGEX =
          
        

        
Regexp.new('\A[\*\#\;\:\ ]+')

      
        
PRE_MARKS_REGEX =
          
        

        
Regexp.new('\A\^\ ')

      
        
DEF_MARKS_REGEX =
          
        

        
Regexp.new('\A[\;\:\ ]+')

      
        
ONSET_BAR_REGEX =
          
        

        
Regexp.new('\A[^\|]+\z')

      
        
CATEGORY_NAMESPACES =
          

  

    
Multilingual category namespace support
Data source: MediaWiki API (siteinfo) via scripts/fetch_mediawiki_data.rb


  





  



        

        
load_mediawiki_data.dig("namespaces", "category")&.join("|") || "Category"

      
        
FILE_NAMESPACES =
          

  

    
Multilingual file namespace support (for image/file links)


  





  



        

        
load_mediawiki_data.dig("namespaces", "file")&.join("|") || "File|Image"

      
        
FILE_NAMESPACES_REGEX =
          
        

        
Regexp.new('\A\s*(?:' + FILE_NAMESPACES + ')\s*:', Regexp::IGNORECASE)

      
        
CATEGORY_NAMESPACE_REGEX =
          

  

    
Multilingual category namespace support (for filtering out category links from body text)


  





  



        

        
Regexp.new('\A\s*(?:' + CATEGORY_NAMESPACES + ')\s*:', Regexp::IGNORECASE)

      
        
IMAGE_PARAM_KEYS =
          

  

    
Image parameters (multilingual) - used for filtering out non-caption parts of File/Image links
Combines: img_thumbnail, img_framed, img_frameless, img_border, img_right, img_left, img_center, img_none,
img_upright, img_baseline, img_sub, img_super, img_top, img_text_top, img_middle, img_bottom, img_text_bottom


  





  



        

        
%w[img_thumbnail img_framed img_frameless img_border img_right img_left img_center img_none
img_upright img_baseline img_sub img_super img_top img_text_top img_middle img_bottom img_text_bottom].freeze

      
        
IMAGE_PARAMS =
          
        

        
IMAGE_PARAM_KEYS.flat_map { |k| load_mediawiki_data.dig("magic_words", k) || [] }.uniq.join("|")

      
        
IMAGE_PARAMS_REGEX =
          
        

        
IMAGE_PARAMS.empty? ? nil : Regexp.new('\A(' + IMAGE_PARAMS + ')\z', Regexp::IGNORECASE)

      
        
CLEANUP_FILE_LINE_REGEX =
          

  

    
Cleanup regex patterns using dynamic file namespaces
For lines like "Image:file.jpg|thumb|...|caption" (gallery/imagemap remnants)


  





  



        

        
Regexp.new('^(?:' + FILE_NAMESPACES + '):[^\n]+\|[^\n]+$', Regexp::IGNORECASE | Regexp::MULTILINE)

      
        
CLEANUP_FILE_INCOMPLETE_REGEX =
          

  

    
For incomplete File/Image links (opened but not closed)


  





  



        

        
Regexp.new('\[\[(?:' + FILE_NAMESPACES + '):[^\]]*\|?\s*$', Regexp::IGNORECASE | Regexp::MULTILINE)

      
        
CATEGORY_REGEX =
          

  

    
Category regex - captures category name without sortkey
[[Category:Name|sortkey]] -> captures only "Name" (not "Name|sortkey")
The (?:[^|]}]*) captures the category name up to | or ] or }


  





  



        

        
Regexp.new('[\{\[\|\b](?:' + CATEGORY_NAMESPACES + ')\s*:([^\|\]\}]+)[\|\]\}]', Regexp::IGNORECASE)

      
        
ESCAPE_NOWIKI_REGEX =
          
        

        
Regexp.new('<nowiki>(.*?)<\/nowiki>', Regexp::MULTILINE)

      
        
UNESCAPE_NOWIKI_REGEX =
          
        

        
Regexp.new('<nowiki\-(\d+?)>')

      
        
REMOVE_ISOLATED_REGEX =
          
        

        
Regexp.new('^\s*\{\{(.*?)\}\}\s*$')

      
        
REMOVE_INLINE_REGEX =
          
        

        
Regexp.new('\{\{(.*?)\}\}')

      
        
SINGLE_SQUARE_BRACKET_REGEX =
          

  

    
Note: TYPE_CODE_REGEX removed (was unused dead code)
Template type detection is now handled by data-driven patterns in template_aliases.json


  





  



        

        
Regexp.new("(#{Regexp.escape("[")}|#{Regexp.escape("]")})", Regexp::MULTILINE)

      
        
DOUBLE_SQUARE_BRACKET_REGEX =
          
        

        
Regexp.new("(#{Regexp.escape("[[")}|#{Regexp.escape("]]")})", Regexp::MULTILINE)

      
        
SINGLE_CURLY_BRACKET_REGEX =
          
        

        
Regexp.new("(#{Regexp.escape("{")}|#{Regexp.escape("}")})", Regexp::MULTILINE)

      
        
DOUBLE_CURLY_BRACKET_REGEX =
          
        

        
Regexp.new("(#{Regexp.escape("{{")}|#{Regexp.escape("}}")})", Regexp::MULTILINE)

      
        
CURLY_SQUARE_BRACKET_REGEX =
          
        

        
Regexp.new("(#{Regexp.escape("{|")}|#{Regexp.escape("|}")})", Regexp::MULTILINE)

      
        
SELF_CLOSING_TAG_REGEX =
          
        

        
Regexp.new('<[^<>]+/>')

      
        
COMPLEX_REGEX_01 =
          
        

        
Regexp.new('\<\<([^<>]++)\>\>\s?')

      
        
COMPLEX_REGEX_02 =
          
        

        
Regexp.new('\[\[File\:((?:[^\[\]]++|\[\[\g<1>\]\])++)\]\]', Regexp::MULTILINE | Regexp::IGNORECASE)

      
        
COMPLEX_REGEX_03 =
          
        

        
Regexp.new('^\[\[((?:[^\[\]]++|\[\[\g<1>\]\])++)^\]\]', Regexp::MULTILINE)

      
        
COMPLEX_REGEX_04 =
          
        

        
Regexp.new('\{\{(?:infobox|efn|sfn|unreliable source|refn|reflist|col(?:umns)?\-list|div col|no col|bar box|formatnum\:|col\||see also\||r\||#)((?:[^{}]++|\{\{\g<1>\}\})++)\}\}', Regexp::MULTILINE | Regexp::IGNORECASE)

      
        
COMPLEX_REGEX_05 =
          
        

        
Regexp.new('\{\{[^{}]+?\n\|((?:[^{}]++|\{\{\g<1>\}\})++)\}\}', Regexp::MULTILINE | Regexp::IGNORECASE)

      
        
CLEANUP_REGEX_01 =
          
        

        
Regexp.new('\[ref\]\s*\[\/ref\]', Regexp::MULTILINE)

      
        
CLEANUP_REGEX_02 =
          
        

        
Regexp.new('^File:.+$')

      
        
CLEANUP_REGEX_03 =
          
        

        
Regexp.new('^\|.*$')

      
        
CLEANUP_REGEX_04 =
          
        

        
Regexp.new('\{\{.*$')

      
        
CLEANUP_REGEX_05 =
          
        

        
Regexp.new('^.*\}\}')

      
        
CLEANUP_REGEX_06 =
          
        

        
Regexp.new('\{\|.*$')

      
        
CLEANUP_REGEX_07 =
          
        

        
Regexp.new('^.*\|\}')

      
        
CLEANUP_REGEX_08 =
          
        

        
Regexp.new('\n\n\n+', Regexp::MULTILINE)

      
        
CLEANUP_MIXED_WHITESPACE_REGEX =
          

  

    
Additional cleanup patterns (pre-compiled for performance)
Mixed whitespace between newlines: \n<spaces/tabs>\n<spaces/tabs>\n+ -> \n\n


  





  



        

        
Regexp.new('\n[ \t]*\n[ \t]*\n+')

      
        
CLEANUP_MULTIPLE_SPACES_REGEX =
          

  

    
Multiple consecutive spaces (not at line start) -> single space


  





  



        

        
Regexp.new('([^\n]) {2,}')

      
        
CLEANUP_EMPTY_PARENS_REGEX =
          

  

    
Empty parentheses (ASCII and Japanese) - combined for single-pass


  





  



        

        
Regexp.new('\(\s*\)|（\s*）')

      
        
CLEANUP_MULTIPLE_PIPES_REGEX =
          

  

    
Multiple pipes (table remnants)


  





  



        

        
Regexp.new('\|\|+')

      
        
CLEANUP_TRAILING_PIPE_REGEX =
          

  

    
Trailing pipe at end of line


  





  



        

        
Regexp.new('\|\s*$')

      
        
CLEANUP_PIPE_LINE_REGEX =
          

  

    
Lines that are just pipe + content (table rows)


  





  



        

        
Regexp.new('^\s*\|[^|]*$\n?', Regexp::MULTILINE)

      
        
CLEANUP_KEY_VALUE_LINE_REGEX =
          

  

    
Lines with multiple pipe-separated key=value pairs (infobox remnants)


  





  



        

        
Regexp.new('^\s*\|?\w+=[\w\s-]+(?:\|\w+=[\w\s-]+)+\s*$', Regexp::MULTILINE)

      
        
CLEANUP_ORPHANED_CLOSE_REGEX =
          

  

    
Orphaned closing brackets (]] at start of line or after whitespace)


  





  



        

        
Regexp.new('(?:^|(?<=\s))([^|\[\]\n]+)\]\]')

      
        
CLEANUP_ORPHANED_OPEN_REGEX =
          

  

    
Orphaned opening wiki brackets not closed on same line


  





  



        

        
Regexp.new('\[\[[^\[\]\n]*$')

      
        
CLEANUP_STANDALONE_CLOSE_REGEX =
          

  

    
Standalone ]] on its own line


  





  



        

        
Regexp.new('^\s*\]\]\s*$', Regexp::MULTILINE)

      
        
CLEANUP_ORPHANED_BRACKETS_REGEX =
          

  

    
Combined pattern for orphaned brackets (both open and standalone close) - single pass removal


  





  



        

        
Regexp.new('\[\[[^\[\]\n]*$|^\s*\]\]\s*$', Regexp::MULTILINE)

      
        
CLEANUP_PIPE_CLOSE_REGEX =
          

  

    
]] preceded by pipe without matching [[ (orphaned from broken links)


  





  



        

        
Regexp.new('([^|\[\]\n])\|([^|\[\]\n]+)\]\](?!\])')

      
        
CLEANUP_MULTI_BLANK_REGEX =
          

  

    
Multiple blank lines (final cleanup)


  





  



        

        
Regexp.new('\n{3,}')

      
        
IMAGEMAP_COORD_REGEX =
          

  

    
Imagemap coordinate remnants (rect, poly, circle, default with coordinates)


  





  



        

        
Regexp.new('^(?:rect|poly|circle|default)\s+[\d\s]+.*$', Regexp::IGNORECASE)

      
        
DEFAULTSORT_KEYWORDS =
          

  

    
MediaWiki magic words (universal across all wikis)
DEFAULTSORT, DISPLAYTITLE, etc. - loaded from mediawiki_aliases.json for multilingual support


  





  



        

        
load_mediawiki_data.dig("magic_words", "defaultsort")&.join("|") || "DEFAULTSORT"

      
        
DISPLAYTITLE_KEYWORDS =
          
        

        
load_mediawiki_data.dig("magic_words", "displaytitle")&.join("|") || "DISPLAYTITLE"

      
        
MAGIC_WORD_LINE_REGEX =
          

  

    
Match bare magic words on their own line: DEFAULTSORT:value or デフォルトソート:value


  





  



        

        
Regexp.new('^(?:' + DEFAULTSORT_KEYWORDS + '|' + DISPLAYTITLE_KEYWORDS + ')[^\n]*$', Regexp::IGNORECASE)

      
        
MAGIC_WORD_TEMPLATE_REGEX =
          

  

    
Match magic word template format: {DEFAULTSORT:value} or {デフォルトソート:value}


  





  



        

        
Regexp.new('\{\{\s*(?:' + DEFAULTSORT_KEYWORDS + '|' + DISPLAYTITLE_KEYWORDS + ')[^\}]*\}\}', Regexp::IGNORECASE)

      
        
DOUBLE_UNDERSCORE_PATTERNS =
          

  

    
Double-underscore magic words: NOTOC, TOC, FORCETOC, NOEDITSECTION, etc.
Data source: MediaWiki API (siteinfo) via scripts/fetch_mediawiki_data.rb
Contains 1198 multilingual aliases for behavior switches


  





  



        

        
load_mediawiki_data.dig("magic_words", "double_underscore") || []

      
        
DOUBLE_UNDERSCORE_MAGIC_REGEX =
          
        

        
if DOUBLE_UNDERSCORE_PATTERNS.empty?
  Regexp.new('__[A-Z]+__')  # Fallback to basic pattern
else
  # Build alternation pattern from actual magic word aliases
  pattern = DOUBLE_UNDERSCORE_PATTERNS.map { |p| Regexp.escape(p) }.join("|")
  Regexp.new('(?:' + pattern + ')', Regexp::IGNORECASE)
end

      
        
REMOVE_DIRECTIVES_REGEX =
          

  

    
Data-driven pattern for removing double-underscore behavior switches from text
Uses the comprehensive multilingual magic word list (1198 aliases)
Falls back to generic pattern if data file is empty


  





  



        

        
DOUBLE_UNDERSCORE_PATTERNS.empty? ?
REMOVE_DIRECTIVES_REGEX_GENERIC :
DOUBLE_UNDERSCORE_MAGIC_REGEX

      
        
INTERWIKI_PREFIX_REGEX =
          

  

    
Interwiki links: :en:Article, :fr:Article, :de:Article, etc.
Removes the prefix but keeps the article name


  





  



        

        
Regexp.new(':([a-z]{2,3}):(?=[^\s\]]+)')

      
        
AUTHORITY_CONTROL_TEMPLATES =
          

  

    
Authority control and metadata templates (standalone lines)
These are template names that appear alone on a line after processing
Data source: template_aliases.json (authority_control category)


  





  



        

        
load_template_data["authority_control"] || []

      
        
AUTHORITY_CONTROL_REGEX =
          
        

        
if AUTHORITY_CONTROL_TEMPLATES.empty?
  # Fallback to basic pattern
  Regexp.new(
    '^\s*(Normdaten|Authority\s*control|Persondata|VIAF|LCCN|GND)\s*$',
    Regexp::MULTILINE | Regexp::IGNORECASE
  )
else
  pattern = AUTHORITY_CONTROL_TEMPLATES.map { |t| Regexp.escape(t) }.join("|")
  Regexp.new('^\s*(' + pattern + ')\s*$', Regexp::MULTILINE | Regexp::IGNORECASE)
end

      
        
CLEANUP_REMNANTS_TEMPLATES =
          

  

    
Cleanup remnants - template names that appear as artifacts after processing
Data source: template_aliases.json (cleanup_remnants category)


  





  



        

        
load_template_data["cleanup_remnants"] || []

      
        
CLEANUP_REMNANTS_REGEX =
          
        

        
if CLEANUP_REMNANTS_TEMPLATES.empty?
  # Fallback to basic pattern
  Regexp.new('^\s*(Clear|Clearleft|Clearright|notelist\d*)\s*$', Regexp::MULTILINE | Regexp::IGNORECASE)
else
  pattern = CLEANUP_REMNANTS_TEMPLATES.map { |t| Regexp.escape(t) }.join("|")
  # Also match notelist with numbers (notelist2, notelist3, etc.)
  pattern += '|notelist\d+'
  Regexp.new('^\s*(' + pattern + ')\s*$', Regexp::MULTILINE | Regexp::IGNORECASE)
end

      
        
CATEGORY_LINE_REGEX =
          

  

    
Category line patterns for all Wikipedia languages
Loaded from mediawiki_aliases.json for complete multilingual support (230+ languages)
Note: Must NOT match "CATEGORIES:" (our summary line)


  





  



        

        
Regexp.new(
  '^\s*\*?\s*(?!CATEGORIES)(?:' + CATEGORY_NAMESPACES + '):[^\n]+$',
  Regexp::MULTILINE | Regexp::IGNORECASE
)

      
        
SISTER_PROJECTS =
          

  

    
Wikimedia sister project markers (standalone lines)
Data source: MediaWiki API (siteinfo interwikimap) via scripts/fetch_mediawiki_data.rb
Contains 546 sister project prefixes from all Wikipedia language editions


  





  



        

        
load_mediawiki_data.dig("interwiki", "sister_projects") || []

      
        
WIKIMEDIA_PROJECT_NAMES =
          

  

    
Filter to only keep known Wikimedia project names (not language codes)


  





  



        

        
%w[
  wikibooks wikiversity wikisource wikiquote wikinews wiktionary
  wikivoyage wikispecies wikidata commons meta mediawiki
  mediawikiwiki species oldwikisource wikifunctions school
].freeze

      
        
WIKIMEDIA_PROJECT_REGEX =
          
        

        
begin
  # Combine known project names with any from data
  projects_from_data = SISTER_PROJECTS.select { |p| WIKIMEDIA_PROJECT_NAMES.include?(p.downcase) }
  # Always include all known project names (ensures complete coverage)
  all_projects = (WIKIMEDIA_PROJECT_NAMES + projects_from_data).uniq
  # Add common variations
  pattern_parts = all_projects.map { |p| Regexp.escape(p) }
  pattern_parts << 'Wikimedia\s*Commons'  # Common alternate form
  pattern_parts << 'Commons\s*cat(?:egory)?'  # Commons category template
  Regexp.new(
    '^\s*(' + pattern_parts.join("|") + ')(?::|$)',
    Regexp::MULTILINE | Regexp::IGNORECASE
  )
end

      
        
LONE_ASTERISK_REGEX =
          

  

    
Lines that are just a single asterisk (list marker without content)


  





  



        

        
Regexp.new('^\s*\*\s*$', Regexp::MULTILINE)

      
        
NON_ARTICLE_NAMESPACES =
          

  

    
=========================================================================


Non-article namespace prefixes (for validation filtering)


These are namespaces that should be excluded from article validation
as they contain templates, portals, help pages, etc. not encyclopedia content


Data source: MediaWiki API (siteinfo) via scripts/fetch_mediawiki_data.rb
Contains 6083 namespace aliases from 351 Wikipedia language editions


  





  



        

        
(load_mediawiki_data.dig("namespaces", "non_article") || []).freeze

      
        
NON_ARTICLE_NAMESPACE_REGEX =
          

  

    
Build regex for matching non-article titles
Matches "Namespace:Title" where Namespace is in the list


  





  



        

        
if NON_ARTICLE_NAMESPACES.empty?
  # Fallback to basic English namespaces
  Regexp.new(
    '\A\s*(Wikipedia|Template|Portal|Help|Category|File|Image|User|Talk|Module|Draft|MediaWiki)\s*:',
    Regexp::IGNORECASE
  )
else
  Regexp.new(
    '\A\s*(' + NON_ARTICLE_NAMESPACES.map { |ns| Regexp.escape(ns) }.join("|") + ')\s*:',
    Regexp::IGNORECASE
  )
end

      
        
RACTOR_SHAREABLE_EXCLUDES =
          

  

    
Constants that should NOT be made Ractor-shareable
(they require mutable state or are already shareable)


  





  



        

        
%i[HTML_DECODER RACTOR_SHAREABLE_EXCLUDES].freeze

      
        
MARKER_TYPES =
          

  

    
Marker types for special content


  





  



        

        
%i[math code chem table score timeline graph ipa infobox navbox gallery sidebar mapframe imagemap references codeblock].freeze

      
        
INLINE_MARKERS =
          

  

    
Inline markers: removing these can break surrounding text


  





  



        

        
%i[math chem ipa code].freeze

      
        
BLOCK_MARKERS =
          

  

    
Block markers: these are standalone and can be safely removed


  





  



        

        
%i[table score timeline graph infobox navbox gallery sidebar mapframe imagemap references codeblock].freeze

      
        
DEFAULT_MARKERS =
          

  

    
Default: all markers enabled


  





  



        

        
MARKER_TYPES.dup.freeze

      
        
MARKER_PATTERNS =
          

  

    
Regex patterns for marker detection


  





  



        

        
{
  # MATH: <math>...</math>, {{math|...}}, {{mvar|...}}
  math: {
    tags: [/<math[^>]*>.*?<\/math>/mi],
    templates: [/\{\{(?:math|mvar)\s*\|/i]
  },
  # CODE: <code>...</code> (inline only)
  code: {
    tags: [
      /<code[^>]*>.*?<\/code>/mi
    ],
    templates: []
  },
  # CODEBLOCK: <syntaxhighlight>...</syntaxhighlight>, <source>...</source>, <pre>...</pre> (block)
  codeblock: {
    tags: [
      /<syntaxhighlight[^>]*>.*?<\/syntaxhighlight>/mi,
      /<source[^>]*>.*?<\/source>/mi,
      /<pre[^>]*>.*?<\/pre>/mi
    ],
    templates: []
  },
  # CHEM: <chem>...</chem>, {{chem|...}}, {{ce|...}}
  chem: {
    tags: [/<chem[^>]*>.*?<\/chem>/mi],
    templates: [/\{\{(?:chem|ce)\s*\|/i]
  },
  # TABLE: {|...|}, <table>...</table>
  table: {
    tags: [/<table[^>]*>.*?<\/table>/mi],
    wiki_table: true
  },
  # SCORE: <score>...</score>
  score: {
    tags: [/<score[^>]*>.*?<\/score>/mi],
    templates: []
  },
  # TIMELINE: <timeline>...</timeline>
  timeline: {
    tags: [/<timeline[^>]*>.*?<\/timeline>/mi],
    templates: []
  },
  # GRAPH: <graph>...</graph>
  graph: {
    tags: [/<graph[^>]*>.*?<\/graph>/mi],
    templates: []
  },
  # IPA: {{IPA|...}}, {{IPAc-en|...}}, etc.
  ipa: {
    tags: [],
    templates: [/\{\{IPA[c]?(?:-[a-z]{2,3})?\s*\|/i]
  },
  # INFOBOX: {{Infobox ...}}
  infobox: {
    tags: [],
    templates: [/\{\{[Ii]nfobox\s*/]
  },
  # NAVBOX: {{Navbox ...}}
  navbox: {
    tags: [],
    templates: [/\{\{[Nn]avbox\s*/]
  },
  # GALLERY: <gallery>...</gallery>
  gallery: {
    tags: [/<gallery[^>]*>.*?<\/gallery>/mi],
    templates: []
  },
  # SIDEBAR: {{Sidebar ...}}
  sidebar: {
    tags: [],
    templates: [/\{\{[Ss]idebar\s*/]
  },
  # MAPFRAME: <mapframe>...</mapframe>
  mapframe: {
    tags: [/<mapframe[^>]*>.*?<\/mapframe>/mi],
    templates: []
  },
  # IMAGEMAP: <imagemap>...</imagemap>
  imagemap: {
    tags: [/<imagemap[^>]*>.*?<\/imagemap>/mi],
    templates: []
  },
  # REFERENCES: {{reflist}}, {{refbegin}}...{{refend}}, <references/>
  references: {
    tags: [
      /<references\s*\/>/mi,
      /<references[^>]*>.*?<\/references>/mi
    ],
    templates: [/\{\{[Rr]eflist\s*/],
    paired_templates: [{ start: /\{\{[Rr]efbegin/i, end_name: "refend" }]
  }
}.freeze

      
        
CITATION_TEMPLATES =
          

  

    
Citation templates that can be extracted
Data source: template_aliases.json (citation_templates category)


  





  



        

        
Wp2txt.load_template_data["citation_templates"] || []

      
        
CITATION_TEMPLATE_REGEX =
          
        

        
if CITATION_TEMPLATES.empty?
  # Fallback to basic pattern
  /\A\s*(?:cite\s*(?:web|book|news|journal)|citation)\s*(?:\||$)/i
else
  pattern = CITATION_TEMPLATES.map { |t| Regexp.escape(t) }.join("|")
  Regexp.new('\A\s*(?:' + pattern + ')\s*(?:\||$)', Regexp::IGNORECASE)
end

      
        
REMOVE_TEMPLATES =
          

  

    
Templates that should be completely removed (references, navigation, but NOT citations when extracting)
Data source: template_aliases.json (remove_templates category)


  





  



        

        
Wp2txt.load_template_data["remove_templates"] || []

      
        
REMOVE_TEMPLATES_REGEX =
          
        

        
if REMOVE_TEMPLATES.empty?
  # Fallback to basic pattern
  /\A\s*(?:sfn|efn|refn|reflist|notelist|main|see\s*also|portal)\s*(?:\||$)/i
else
  pattern = REMOVE_TEMPLATES.map { |t| Regexp.escape(t) }.join("|")
  Regexp.new('\A\s*(?:' + pattern + ')\s*(?:\||$)', Regexp::IGNORECASE)
end

      
        
FLAG_TEMPLATES =
          

  

    
Flag templates to remove
Data source: template_aliases.json (flag_templates category)


  





  



        

        
Wp2txt.load_template_data["flag_templates"] || []

      
        
FLAG_TEMPLATE_REGEX =
          
        

        
if FLAG_TEMPLATES.empty?
  /\A\s*(?:flag|flagicon|flagcountry)\s*(?:\||$)/i
else
  pattern = FLAG_TEMPLATES.map { |t| Regexp.escape(t) }.join("|")
  Regexp.new('\A\s*(?:' + pattern + ')\s*(?:\||$)', Regexp::IGNORECASE)
end

      
        
FORMATTING_TEMPLATES =
          

  

    
Formatting templates (extract content)
Data source: template_aliases.json (formatting_templates category)


  





  



        

        
Wp2txt.load_template_data["formatting_templates"] || []

      
        
FORMATTING_TEMPLATE_REGEX =
          
        

        
if FORMATTING_TEMPLATES.empty?
  /\A\s*(?:small|smaller|large|larger|nowrap|nbsp)\s*(?:\||$)/i
else
  pattern = FORMATTING_TEMPLATES.map { |t| Regexp.escape(t) }.join("|")
  Regexp.new('\A\s*(?:' + pattern + ')\s*(?:\||$)', Regexp::IGNORECASE)
end

      
        
RUBY_TEXT_TEMPLATES =
          

  

    
Ruby text templates (読み仮名 equivalent across languages)
Data source: template_aliases.json (ruby_text_templates category)


  





  



        

        
Wp2txt.load_template_data["ruby_text_templates"] || []

      
        
        
Wp2txt.load_template_data["interwiki_link_templates"] || []

      
        
MIXED_SCRIPT_TEMPLATES =
          

  

    
Mixed script templates (nihongo equivalent across languages)
Data source: template_aliases.json (mixed_script_templates category)


  





  



        

        
Wp2txt.load_template_data["mixed_script_templates"] || []

      
        
CONVERT_TEMPLATES =
          

  

    
Convert templates
Data source: template_aliases.json (convert_templates category)


  





  



        

        
Wp2txt.load_template_data["convert_templates"] || []

      
        
COUNTRY_CODE_REGEX =
          

  

    
Country code templates (2-3 letter codes that represent flags)


  





  



        

        
/\A[A-Z]{2,3}\z/

      
        
VERSION =
          
        

        
"2.1.2"

      
        
SECONDS_PER_DAY =
          

  

    


Time Constants


  





  



        

        
86_400

      
        
SECONDS_PER_HOUR =
          
        

        
3_600

      
        
SECONDS_PER_MINUTE =
          
        

        
60

      
        
DEFAULT_DUMP_EXPIRY_DAYS =
          

  

    


Cache Settings


Default expiry for downloaded Wikipedia dump files


  





  



        

        
30

      
        
DEFAULT_CATEGORY_CACHE_EXPIRY_DAYS =
          

  

    
Default expiry for category member cache


  





  



        

        
7

      
        
DEFAULT_HTTP_TIMEOUT =
          

  

    


Network Settings


Default timeout for HTTP requests (seconds)


  





  



        

        
30

      
        
DEFAULT_PROGRESS_INTERVAL =
          

  

    
Default progress reporting interval (seconds)


  





  



        

        
10

      
        
INDEX_PROGRESS_THRESHOLD =
          

  

    
Index parsing progress reporting threshold (entries)


  





  



        

        
500_000

      
        
DEFAULT_TOP_N_SECTIONS =
          

  

    
Default number of top section headings to include in stats output


  





  



        

        
50

      
        
RESUME_METADATA_MAX_AGE_DAYS =
          

  

    
Download resume metadata max age (days)


  





  



        

        
7

      
        
MAX_NESTING_ITERATIONS =
          

  

    


Processing Limits


Safety limit for deeply nested structure processing (templates, tables, etc.)
This prevents infinite loops in malformed markup


  





  



        

        
50_000

      
        
DEFAULT_BUFFER_SIZE =
          

  

    
Buffer size for file reading (10 MB)
Optimized for Wikipedia dump processing


  





  



        

        
10_485_760

      
        
MIN_BUFFER_SIZE =
          

  

    
Minimum buffer size (1 MB) - don't go below this


  





  



        

        
1_048_576

      
        
MAX_BUFFER_SIZE =
          

  

    
Maximum buffer size (100 MB) - don't exceed this


  





  



        

        
104_857_600

      
        
BYTES_PER_KB =
          

  

    


File Size Units (Binary - for accurate file sizes)


  





  



        

        
1_024

      
        
BYTES_PER_MB =
          
        

        
1_024 * 1_024

      
        
BYTES_PER_GB =
          
        

        
1_024 * 1_024 * 1_024

      
        
MAX_HTTP_RETRIES =
          

  

    
Maximum number of retries for transient network errors


  





  



        

        
3

      
        
EXTENSION_TAGS =
          

  

    
Extension tags to remove (block-level tags that should be stripped)
Data source: mediawiki_aliases.json (extension_tags)
These are MediaWiki extension tags like , , , etc.


  





  



        

        
Wp2txt.load_mediawiki_data["extension_tags"] || []

      
        
BLOCK_EXTENSION_TAGS =
          

  

    
Block-level extension tags to process in remove_html
Not all extension tags should be removed here - some are handled by markers (math, chem, etc.)
and some are inline (ref). We only remove block-level content containers.


  





  



        

        
%w[div gallery timeline noinclude imagemap poem hiero graph categorytree section].freeze

      
    

  




  
Class Attribute Summary collapse

  




  
    

      Class Method Summary
      collapse
    


    
    
  
    

      Instance Method Summary
      collapse
    


    
    
  


  

    
Class Attribute Details

    
      
      
      

  

  
    .regex_cache  ⇒ Object 
  

  

  


  

    
Returns the value of attribute regex_cache.


  





  



  

    
      


12
13
14

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 12

def regex_cache
  @regex_cache
end

    		
  





    
  



  

    
Class Method Details


    
      

  

  
    .article_page?(title)  ⇒ Boolean 
  

  

  


  

    
Helper method to check if a title is an article page (not a special namespace)


  





  

Returns:


    
  
    
  • 
      
      
        (Boolean)
      
      
      
    
    • 
  





  

    
      


442
443
444
445

    		
    
      
# File 'lib/wp2txt/regex.rb', line 442

def self.article_page?(title)
  return true if title.nil? || title.empty?
  !(title =~ NON_ARTICLE_NAMESPACE_REGEX)
end

    		
  





    
      

  

  
    .build_extra_entities_regex  ⇒ Object 
  

  

  


  

    
Build regex for extra entities not handled by HTMLEntities gem


  





  



  

    
      


135
136
137
138
139
140
141
142

    		
    
      
# File 'lib/wp2txt/regex.rb', line 135

def self.build_extra_entities_regex
  entities = load_html_entities
  return nil if entities.empty?

  # Build regex pattern for all entity keys
  pattern = "(" + entities.keys.map { |k| Regexp.escape(k) }.join("|") + ")"
  Regexp.new(pattern)
end

    		
  





    
      

  

  
    .build_template_regex(templates, options = {})  ⇒ Object 
  

  

  


  

    
Build regex pattern from template list (escapes special chars, joins with |)


  





  



  

    
      


87
88
89
90
91
92
93
94
95
96
97
98
99
100

    		
    
      
# File 'lib/wp2txt/regex.rb', line 87

def self.build_template_regex(templates, options = {})
  return nil if templates.nil? || templates.empty?

  pattern = templates.map { |t| Regexp.escape(t) }.join("|")
  if options[:anchor_start]
    pattern = '\A\s*(?:' + pattern + ')'
  else
    pattern = '(?:' + pattern + ')'
  end
  if options[:require_pipe_or_end]
    pattern += '\s*(?:\||$)'
  end
  Regexp.new(pattern, Regexp::IGNORECASE)
end

    		
  





    
      

  

  
    .days_to_seconds(days)  ⇒ Integer 
  

  

  


  

    
Convert days to seconds


  





  
Parameters:


    
  
    
  • 
      
        days
      
      
        (Integer, Float)
      
      
      
        —
        
    Number of days
    
      
    
    • 
  




Returns:


    
  
    
  • 
      
      
        (Integer)
      
      
      
        —
        
    Seconds
    
      
    
    • 
  





  

    
      


97
98
99

    		
    
      
# File 'lib/wp2txt/constants.rb', line 97

def days_to_seconds(days)
  (days * SECONDS_PER_DAY).to_i
end

    		
  





    
      

  

  
    .file_age_days(path)  ⇒ Float? 
  

  

  


  

    
Calculate file age in days


  





  
Parameters:


    
  
    
  • 
      
        path
      
      
        (String)
      
      
      
        —
        
    File path
    
      
    
    • 
  




Returns:


    
  
    
  • 
      
      
        (Float, nil)
      
      
      
        —
        
    Age in days, or nil if file doesn't exist
    
      
    
    • 
  





  

    
      


114
115
116
117
118

    		
    
      
# File 'lib/wp2txt/constants.rb', line 114

def file_age_days(path)
  return nil unless File.exist?(path)

  ((Time.now - File.mtime(path)) / SECONDS_PER_DAY).round(1)
end

    		
  





    
      

  

  
    .file_fresh?(path, days)  ⇒ Boolean 
  

  

  


  

    
Check if a file is older than specified days


  





  
Parameters:


    
  
    
  • 
      
        path
      
      
        (String)
      
      
      
        —
        
    File path
    
      
    
    • 
  
    
  • 
      
        days
      
      
        (Integer)
      
      
      
        —
        
    Number of days
    
      
    
    • 
  




Returns:


    
  
    
  • 
      
      
        (Boolean)
      
      
      
        —
        
    true if file is fresh (not expired)
    
      
    
    • 
  





  

    
      


105
106
107
108
109

    		
    
      
# File 'lib/wp2txt/constants.rb', line 105

def file_fresh?(path, days)
  return false unless File.exist?(path)

  File.mtime(path) > Time.now - days_to_seconds(days)
end

    		
  





    
      

  

  
    .format_file_size(bytes)  ⇒ String 
  

  

  


  

    
Format file size in human-readable form (binary units)


  





  
Parameters:


    
  
    
  • 
      
        bytes
      
      
        (Integer)
      
      
      
        —
        
    Size in bytes
    
      
    
    • 
  




Returns:


    
  
    
  • 
      
      
        (String)
      
      
      
        —
        
    Formatted size (e.g., "1.5 MB")
    
      
    
    • 
  





  

    
      


123
124
125
126
127
128
129
130
131
132
133

    		
    
      
# File 'lib/wp2txt/constants.rb', line 123

def format_file_size(bytes)
  if bytes < BYTES_PER_KB
    "#{bytes} B"
  elsif bytes < BYTES_PER_MB
    "#{(bytes.to_f / BYTES_PER_KB).round(1)} KB"
  elsif bytes < BYTES_PER_GB
    "#{(bytes.to_f / BYTES_PER_MB).round(1)} MB"
  else
    "#{(bytes.to_f / BYTES_PER_GB).round(2)} GB"
  end
end

    		
  





    
      

  

  
    .load_html_entities  ⇒ Object 
  

  

  


  

    
Load HTML entities from WHATWG data file (generated by scripts/fetch_html_entities.rb)
Uses SQLite cache for faster subsequent loads


  





  



  

    
      


104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

    		
    
      
# File 'lib/wp2txt/regex.rb', line 104

def self.load_html_entities
  return @html_entities if @html_entities

  # Try SQLite cache first (combined entities)
  cached = GlobalDataCache.load(:html_entities_combined)
  if cached
    @html_entities = cached
    return @html_entities
  end

  @html_entities = {}

  # Load WHATWG HTML entities
  if File.exist?(HTML_ENTITIES_PATH)
    data = JSON.parse(File.read(HTML_ENTITIES_PATH))
    @html_entities.merge!(data["entities"] || {})
  end

  # Load Wikipedia-specific entities (override/supplement WHATWG)
  if File.exist?(WIKIPEDIA_ENTITIES_PATH)
    data = JSON.parse(File.read(WIKIPEDIA_ENTITIES_PATH))
    @html_entities.merge!(data["entities"] || {})
  end

  # Save combined entities to cache
  GlobalDataCache.save(:html_entities_combined, @html_entities) unless @html_entities.empty?

  @html_entities
end

    		
  





    
      

  

  
    .load_mediawiki_data  ⇒ Object 
  

  

  


  

    
Load MediaWiki aliases from data file (generated by scripts/fetch_mediawiki_data.rb)
Uses SQLite cache for faster subsequent loads


  





  



  

    
      


33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

    		
    
      
# File 'lib/wp2txt/regex.rb', line 33

def self.load_mediawiki_data
  return @mediawiki_data if @mediawiki_data

  # Try SQLite cache first
  cached = GlobalDataCache.load(:mediawiki)
  if cached
    @mediawiki_data = cached
    return @mediawiki_data
  end

  # Load from JSON file
  if File.exist?(MEDIAWIKI_DATA_PATH)
    @mediawiki_data = JSON.parse(File.read(MEDIAWIKI_DATA_PATH))
    # Save to cache for future use
    GlobalDataCache.save(:mediawiki, @mediawiki_data)
  else
    # Fallback to minimal defaults if data file is missing
    @mediawiki_data = {
      "magic_words" => { "redirect" => ["REDIRECT"] },
      "namespaces" => { "category" => ["Category"], "file" => ["File", "Image"] }
    }
  end
  @mediawiki_data
end

    		
  





    
      

  

  
    .load_template_data  ⇒ Object 
  

  

  


  

    
Load template aliases from data file (generated by scripts/fetch_template_data.rb)
Uses SQLite cache for faster subsequent loads


  





  



  

    
      


60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

    		
    
      
# File 'lib/wp2txt/regex.rb', line 60

def self.load_template_data
  return @template_data if @template_data

  # Try SQLite cache first
  cached = GlobalDataCache.load(:template)
  if cached
    @template_data = cached
    return @template_data
  end

  # Load from JSON file
  if File.exist?(TEMPLATE_DATA_PATH)
    @template_data = JSON.parse(File.read(TEMPLATE_DATA_PATH))
    # Save to cache for future use
    GlobalDataCache.save(:template, @template_data)
  else
    # Fallback to minimal defaults if data file is missing
    @template_data = {
      "remove_templates" => %w[reflist notelist main see\ also portal],
      "authority_control" => %w[authority\ control normdaten],
      "cleanup_remnants" => %w[clear clearleft clearright]
    }
  end
  @template_data
end

    		
  





    
      

  

  
    .make_constants_ractor_shareable!  ⇒ Object 
  

  

  


  

    
      


457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472

    		
    
      
# File 'lib/wp2txt/regex.rb', line 457

def self.make_constants_ractor_shareable!
  return unless defined?(Ractor) && Ractor.respond_to?(:make_shareable)

  constants(false).each do |const_name|
    next if RACTOR_SHAREABLE_EXCLUDES.include?(const_name)

    const = const_get(const_name)
    next if Ractor.shareable?(const)

    begin
      Ractor.make_shareable(const)
    rescue Ractor::IsolationError, FrozenError, TypeError
      # Some constants can't be made shareable, skip them
    end
  end
end

    		
  





    
      

  

  
    .ssl_safe_get(uri, timeout: DEFAULT_HTTP_TIMEOUT, retries: MAX_HTTP_RETRIES)  ⇒ Net::HTTPResponse 
  

  

  


  

    
HTTPS-aware HTTP GET helper with proper SSL verification and retry


  





  
Parameters:


    
  
    
  • 
      
        uri
      
      
        (URI)
      
      
      
        —
        
    The URI to request
    
      
    
    • 
  
    
  • 
      
        timeout
      
      
        (Integer)
      
      
        (defaults to: DEFAULT_HTTP_TIMEOUT)
      
      
        —
        
    Timeout in seconds
    
      
    
    • 
  
    
  • 
      
        retries
      
      
        (Integer)
      
      
        (defaults to: MAX_HTTP_RETRIES)
      
      
        —
        
    Maximum number of retries on transient errors
    
      
    
    • 
  




Returns:


    
  
    
  • 
      
      
        (Net::HTTPResponse)
      
      
      
        —
        
    The HTTP response
    
      
    
    • 
  





  

    
      


23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

    		
    
      
# File 'lib/wp2txt/multistream.rb', line 23

def self.ssl_safe_get(uri, timeout: DEFAULT_HTTP_TIMEOUT, retries: MAX_HTTP_RETRIES)
  attempts = 0
  begin
    attempts += 1
    http = Net::HTTP.new(uri.host, uri.port)
    http.use_ssl = (uri.scheme == "https")
    http.open_timeout = timeout
    http.read_timeout = timeout

    if http.use_ssl?
      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
    end

    request = Net::HTTP::Get.new(uri)
    http.request(request)
  rescue Net::OpenTimeout, Net::ReadTimeout, SocketError, Errno::ECONNRESET,
         Errno::ECONNREFUSED, Errno::EHOSTUNREACH, OpenSSL::SSL::SSLError => e
    if attempts <= retries
      delay = 2**attempts # Exponential backoff: 2, 4, 8 seconds
      warn "  Network error (attempt #{attempts}/#{retries + 1}): #{e.message}. Retrying in #{delay}s..."
      sleep delay
      retry
    end
    raise
  end
end

    		
  





    
  


  

    
Instance Method Details


    
      

  

  
    #apply_markers(str, enabled_markers)  ⇒ Object 
  

  

  


  

    
Apply marker replacements for enabled marker types
When markers are disabled, content is removed (not marked)


  





  



  

    
      


244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281

    		
    
      
# File 'lib/wp2txt/utils.rb', line 244

def apply_markers(str, enabled_markers)
  result = +str.to_s

  MARKER_PATTERNS.each do |marker_type, patterns|
    placeholder = marker_placeholder(marker_type)
    should_mark = enabled_markers.include?(marker_type)

    # Process HTML-style tags
    patterns[:tags]&.each do |tag_regex|
      if should_mark
        result.gsub!(tag_regex, placeholder)
      else
        # Remove content when marker is not enabled
        result.gsub!(tag_regex, "")
      end
    end

    # Process wiki tables specially (need nested handling)
    if patterns[:wiki_table] && result.include?("{|")
      if should_mark
        result = replace_wiki_table_with_marker(result, placeholder)
      end
      # If not marking, remove_table will handle it later
    end

    # Process template-based markers (Infobox, Navbox, Sidebar)
    patterns[:templates]&.each do |template_regex|
      result = replace_template_with_marker(result, template_regex, placeholder, should_mark)
    end

    # Process paired templates (refbegin...refend)
    patterns[:paired_templates]&.each do |pair|
      result = replace_paired_templates_with_marker(result, pair[:start], pair[:end_name], placeholder, should_mark)
    end
  end

  result
end

    		
  





    
      

  

  
    #apply_pipe_trick(target)  ⇒ Object 
  

  

  


  

    
MediaWiki pipe trick: extracts display text from link target
[[Wikipedia:著作権|]] → 著作権
[[東京 (曖昧さ回避)|]] → 東京


  





  



  

    
      


407
408
409
410
411
412
413
414
415
416

    		
    
      
# File 'lib/wp2txt/utils.rb', line 407

def apply_pipe_trick(target)
  result = target.dup
  # Remove namespace prefix (everything before and including the last colon)
  result = result.sub(/\A[^:]+:/, "") if result.include?(":")
  # Remove trailing parenthetical (disambiguation)
  result = result.sub(/\s*\([^)]+\)\s*\z/, "")
  # Remove trailing comma and following text (for names like "LastName, FirstName")
  result = result.sub(/\s*,.*\z/, "")
  result.strip
end

    		
  





    
      

  

  
    #batch_file_mod(dir_path)  ⇒ Object 
  

  

  


  

    
Modify files under a directory (recursive)


  





  



  

    
      


40
41
42
43
44
45
46
47
48

    		
    
      
# File 'lib/wp2txt/file_utils.rb', line 40

def batch_file_mod(dir_path)
  if FileTest.directory?(dir_path)
    collect_files(dir_path).each do |file|
      yield file if FileTest.file?(file)
    end
  elsif FileTest.file?(dir_path)
    yield dir_path
  end
end

    		
  





    
      

  

  
    #chrref_to_utf(num_str)  ⇒ Object 
  

  

  


  

    
      


49
50
51
52
53
54
55
56
57
58
59
60
61
62

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 49

def chrref_to_utf(num_str)
  num_str.gsub(CHRREF_TO_UTF_REGEX) do
    codepoint = $1 == "x" ? $2.to_i(16) : $2.to_i
    # Handle all valid Unicode codepoints (U+0001 to U+10FFFF)
    if codepoint > 0 && codepoint <= 0x10FFFF
      [codepoint].pack("U")
    else
      ""
    end
  end
rescue RangeError, ArgumentError
  # RangeError: invalid codepoint, ArgumentError: pack error
  num_str
end

    		
  





    
      

  

  
    #cleanup(text)  ⇒ Object 
  

  

  


  

    
cleanup and removal methods ####################


  





  



  

    
      


155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 155

def cleanup(text)
  # Work with a mutable copy to reduce intermediate string allocations
  result = +text.to_s
  result.gsub!(CLEANUP_REGEX_01, "")
  result.gsub!(CLEANUP_REGEX_02, "")
  result.gsub!(CLEANUP_REGEX_03, "")
  result.gsub!(CLEANUP_REGEX_04, "")
  result.gsub!(CLEANUP_REGEX_05, "")
  result.gsub!(CLEANUP_REGEX_06, "")
  result.gsub!(CLEANUP_REGEX_07, "")
  # Reduce 3+ consecutive newlines to 2
  result.gsub!(CLEANUP_REGEX_08, "\n\n")
  # Also handle mixed whitespace patterns (spaces/tabs between newlines)
  result.gsub!(CLEANUP_MIXED_WHITESPACE_REGEX, "\n\n")

  # Fix 1: Multiple consecutive spaces → single space (but preserve indentation at line start)
  result.gsub!(CLEANUP_MULTIPLE_SPACES_REGEX, '\1 ')

  # Fix 2: Empty parentheses → remove (both ASCII and Japanese)
  result.gsub!(CLEANUP_EMPTY_PARENS_REGEX, "")

  # Fix 3: Leftover pipe characters (table/infobox remnants)
  result.gsub!(CLEANUP_MULTIPLE_PIPES_REGEX, "")
  result.gsub!(CLEANUP_TRAILING_PIPE_REGEX, "")
  result.gsub!(CLEANUP_PIPE_LINE_REGEX, "")
  # Lines with multiple pipe-separated key=value pairs (infobox remnants)
  result.gsub!(CLEANUP_KEY_VALUE_LINE_REGEX, "")
  # Template name remnants (data-driven from template_aliases.json)
  result.gsub!(CLEANUP_REMNANTS_REGEX, "")
  # Imagemap/gallery remnants: lines like "Image:file.jpg|thumb|...|caption" without [[ brackets
  result.gsub!(CLEANUP_FILE_LINE_REGEX, "")
  # Incomplete File/Image links (opened but not closed on same logical unit)
  result.gsub!(CLEANUP_FILE_INCOMPLETE_REGEX, "")
  # Orphaned closing brackets from split File links (e.g., "caption]] rest of text")
  result.gsub!(CLEANUP_ORPHANED_CLOSE_REGEX, '\1')
  # Orphaned opening brackets and standalone ]] lines (combined for single pass)
  result.gsub!(CLEANUP_ORPHANED_BRACKETS_REGEX, "")
  # ]] preceded by pipe without matching [[ (orphaned from broken links)
  result.gsub!(CLEANUP_PIPE_CLOSE_REGEX) { "#{$1}#{$2}" }

  # =========================================================================
  # Multilingual cleanup (language-agnostic patterns)
  # =========================================================================

  # MediaWiki magic words: DEFAULTSORT:..., DISPLAYTITLE:...
  # Handles both bare format (DEFAULTSORT:value) and template format ({{DEFAULTSORT:value}})
  result.gsub!(MAGIC_WORD_TEMPLATE_REGEX, "")
  result.gsub!(MAGIC_WORD_LINE_REGEX, "")

  # Double-underscore magic words: __NOTOC__, __TOC__, __FORCETOC__, etc.
  result.gsub!(DOUBLE_UNDERSCORE_MAGIC_REGEX, "")

  # Interwiki links: :en:Article → Article (keep article name, remove prefix)
  result.gsub!(INTERWIKI_PREFIX_REGEX, "")

  # Authority control templates: Normdaten, Authority control, Persondata, etc.
  result.gsub!(AUTHORITY_CONTROL_REGEX, "")

  # Category lines in various languages (but NOT "CATEGORIES:" summary line)
  result.gsub!(CATEGORY_LINE_REGEX, "")

  # Wikimedia sister project markers: Wikibooks, Commons, School:..., etc.
  result.gsub!(WIKIMEDIA_PROJECT_REGEX, "")

  # Lone asterisk lines (list markers without content)
  result.gsub!(LONE_ASTERISK_REGEX, "")

  # Final cleanup: reduce multiple blank lines again after all removals
  result.gsub!(CLEANUP_MULTI_BLANK_REGEX, "\n\n")

  result.strip!
  result << "\n\n"
end

    		
  





    
      

  

  
    #collect_files(str, regex = nil)  ⇒ Object 
  

  

  


  

    
Collect filenames recursively


  





  



  

    
      


9
10
11
12
13
14
15
16

    		
    
      
# File 'lib/wp2txt/file_utils.rb', line 9

def collect_files(str, regex = nil)
  regex ||= //
  text_array = []
  Find.find(str) do |f|
    text_array << f if regex =~ f
  end
  text_array.sort
end

    		
  





    
      

  

  
    #convert_characters(text, _has_retried = false)  ⇒ Object 
  

  

  


  

    
      


15
16
17
18
19
20
21
22
23
24

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 15

def convert_characters(text, _has_retried = false)
  # Use scrub to safely handle invalid byte sequences
  text = text.to_s.scrub("")
  text = chrref_to_utf(text)
  text = special_chr(text)
  text.encode("UTF-8", "UTF-8", invalid: :replace, replace: "")
rescue ::Encoding::InvalidByteSequenceError, ::Encoding::UndefinedConversionError, ArgumentError
  # If any encoding error persists, scrub again and return
  text.to_s.scrub("")
end

    		
  





    
      

  

  
    #correct_inline_template(str, enabled_markers = [], extract_citations = false)  ⇒ Object 
  

  

  


  

    
      


565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679

    		
    
      
# File 'lib/wp2txt/utils.rb', line 565

def correct_inline_template(str, enabled_markers = [], extract_citations = false)
  # Early exit if no templates present
  return str unless str.include?("{{")

  process_nested_single_pass(str, "{{", "}}") do |contents|
    parts = contents.split("|")
    template_name = (parts[0] || "").strip
    template_name_lower = template_name.downcase

    # =========================================================================
    # Specific template handlers (order matters - check before generic patterns)
    # =========================================================================

    # {{IPA|...}} or {{IPA-xx|...}} or {{IPAc-xx|...}}
    # Must be checked BEFORE mixed_script_templates which also contains IPA
    if template_name_lower == "ipa" || template_name_lower.start_with?("ipa-") || template_name_lower.start_with?("ipac-")
      if enabled_markers.include?(:ipa)
        marker_placeholder(:ipa)
      else
        (parts[1] || "").to_s.strip
      end
    # Language templates: {{lang|code|text}} or {{lang-xx|text}}
    # Must be checked BEFORE mixed_script_templates which also contains lang
    elsif template_name_lower == "lang"
      parts.size >= 3 ? parts[2].to_s.strip : (parts[1] || "").to_s.strip
    elsif template_name_lower.start_with?("lang-")
      (parts[1] || "").to_s.strip
    elsif template_name_lower == "fontsize"
      parts.size >= 3 ? parts[2].to_s.strip : (parts[1] || "").to_s.strip
    # {{langwithname|code|name|text}} - extract the text (3rd param)
    elsif template_name_lower == "langwithname"
      parts.size >= 4 ? parts[3].to_s.strip : (parts.last || "").to_s.strip
    # {{math|...}} or {{mvar|...}} - mathematical notation
    elsif template_name_lower == "math" || template_name_lower == "mvar"
      if enabled_markers.include?(:math)
        marker_placeholder(:math)
      else
        (parts[1] || "").to_s.strip
      end
    # {{chem|...}} or {{ce|...}} - chemical formulas
    elsif template_name_lower == "chem" || template_name_lower == "ce"
      if enabled_markers.include?(:chem)
        marker_placeholder(:chem)
      else
        (parts[1] || "").to_s.strip
      end

    # =========================================================================
    # Data-driven template matching (generic patterns from template_aliases.json)
    # =========================================================================

    # Handle citation templates
    elsif CITATION_TEMPLATE_REGEX.match?(contents)
      if extract_citations
        format_citation(contents)
      else
        ""
      end
    # Remove navigation/reference templates entirely
    elsif REMOVE_TEMPLATES_REGEX.match?(contents)
      ""
    # Remove flag templates (data-driven)
    elsif FLAG_TEMPLATE_REGEX.match?(contents) || COUNTRY_CODE_REGEX.match?(template_name)
      ""
    # Ruby text templates: 読み仮名, ruby, etc. (data-driven)
    elsif template_matches?(template_name, RUBY_TEXT_TEMPLATES)
      text = (parts[1] || "").strip
      reading = (parts[2] || "").strip
      reading.empty? ? text : "#{text}（#{reading}）"
    # Interwiki link templates: 仮リンク, ill, interlanguage link (data-driven)
    elsif template_matches?(template_name, INTERWIKI_LINK_TEMPLATES)
      # First parameter is display text
      (parts[1] || "").to_s.strip
    # Mixed script templates: nihongo, transl, etc. (data-driven)
    elsif template_matches?(template_name, MIXED_SCRIPT_TEMPLATES)
      # Format depends on template type
      if template_name_lower == "nihongo" || template_name_lower.start_with?("nihongo")
        text = (parts[1] || "").strip
        kanji = (parts[2] || "").strip
        romaji = (parts[3] || "").strip
        if kanji.empty? && romaji.empty?
          text
        elsif romaji.empty?
          "#{text} (#{kanji})"
        elsif kanji.empty?
          "#{text} (#{romaji})"
        else
          "#{text} (#{kanji}, #{romaji})"
        end
      elsif template_name_lower == "transl" || template_name_lower == "transliteration"
        # {{transl|lang|text}} -> text
        (parts[2] || parts[1] || "").to_s.strip
      else
        # Default: extract first content parameter
        (parts[1] || "").to_s.strip
      end
    # Convert templates (data-driven)
    elsif template_matches?(template_name, CONVERT_TEMPLATES)
      num = (parts[1] || "").strip
      unit = (parts[2] || "").strip
      unit.empty? ? num : "#{num} #{unit}"
    # Formatting templates: small, nowrap, nbsp, etc. (data-driven)
    elsif FORMATTING_TEMPLATE_REGEX.match?(contents)
      if template_name_lower == "nbsp"
        " "  # Non-breaking space
      else
        # Extract content from formatting template
        (parts[1] || "").to_s.strip
      end
    # Default handling for other templates
    else
      extract_template_content(parts)
    end
  end
end

    		
  





    
      

  

  
    #correct_separator(input)  ⇒ Object 
  

  

  


  

    
Take care of difference of separators among environments


  





  



  

    
      


51
52
53
54
55
56
57
58
59
60
61
62
63

    		
    
      
# File 'lib/wp2txt/file_utils.rb', line 51

def correct_separator(input)
  case input
  when String
    # Use tr instead of gsub for simple character replacement (faster)
    if RUBY_PLATFORM.index("win32")
      input.tr("/", "\\")
    else
      input.tr("\\", "/")
    end
  when Array
    input.map { |item| correct_separator(item) }
  end
end

    		
  





    
      

  

  
    #escape_nowiki(str)  ⇒ Object 
  

  

  


  

    
nowiki handling ####################


  





  



  

    
      


132
133
134
135
136
137
138
139
140
141
142
143
144

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 132

def escape_nowiki(str)
  if @nowikis
    @nowikis.clear
  else
    @nowikis = {}
  end
  str.gsub(ESCAPE_NOWIKI_REGEX) do
    nowiki = $1
    nowiki_id = nowiki.object_id
    @nowikis[nowiki_id] = nowiki
    "<nowiki-#{nowiki_id}>"
  end
end

    		
  





    
      

  

  
    #extract_template_content(parts)  ⇒ Object 
  

  

  


  

    
Extract meaningful content from template parts


  





  



  

    
      


682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705

    		
    
      
# File 'lib/wp2txt/utils.rb', line 682

def extract_template_content(parts)
  return "" if parts.empty?
  return parts[0].to_s.strip if parts.size == 1

  # Skip the template name, try to find non-parameter content
  parts[1..].each do |part|
    next if part.nil?
    # Skip if it looks like a parameter (contains =)
    next if part.include?("=")
    content = part.strip
    return content unless content.empty?
  end

  # If all parts have =, try to extract value from first parameter
  parts[1..].each do |part|
    next if part.nil?
    if part.include?("=")
      key, value = part.split("=", 2)
      return value.to_s.strip unless value.nil? || value.strip.empty?
    end
  end

  ""
end

    		
  





    
      

  

  
    #file_mod(file_path, backup = false)  ⇒ Object 
  

  

  


  

    
Modify a file using block/yield mechanism


  





  



  

    
      


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

    		
    
      
# File 'lib/wp2txt/file_utils.rb', line 19

def file_mod(file_path, backup = false)
  str = File.read(file_path)
  newstr = yield(str)
  str = newstr unless newstr.nil?

  require "tempfile"
  dir = File.dirname(file_path)
  temp = Tempfile.new(["wp2txt_", File.extname(file_path)], dir)
  begin
    temp.write(str)
    temp.close
    File.rename(file_path, file_path + ".bak")
    File.rename(temp.path, file_path)
    File.unlink(file_path + ".bak") unless backup
  rescue StandardError
    temp.close! rescue nil # rubocop:disable Style/RescueModifier
    raise
  end
end

    		
  





    
      

  

  
    #finalize_markers(str)  ⇒ Object 
  

  

  


  

    
Convert marker placeholders to final [MARKER] format


  





  



  

    
      


232
233
234
235
236
237
238
239
240

    		
    
      
# File 'lib/wp2txt/utils.rb', line 232

def finalize_markers(str)
  result = +str.to_s
  MARKER_TYPES.each do |marker_type|
    placeholder = marker_placeholder(marker_type)
    final_marker = "[#{marker_type.to_s.upcase}]"
    result.gsub!(placeholder, final_marker)
  end
  result
end

    		
  





    
      

  

  
    #format_citation(contents)  ⇒ Object 
  

  

  


  

    
Extract formatted citation from template parameters


  





  



  

    
      


525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556

    		
    
      
# File 'lib/wp2txt/utils.rb', line 525

def format_citation(contents)
  params = {}
  contents.split("|").each do |part|
    if part.include?("=")
      key, value = part.split("=", 2)
      params[key.strip.downcase] = value&.strip
    end
  end

  # Extract author (last name, or author field)
  author = params["last"] || params["last1"] || params["author"] || params["author1"] || ""
  first = params["first"] || params["first1"] || ""
  author = "#{author}, #{first}" if !author.empty? && !first.empty?

  # Extract title
  title = params["title"] || ""

  # Extract year/date
  year = params["year"] || ""
  if year.empty? && params["date"]
    # Extract year from date like "2021-05-15"
    year = params["date"][0, 4] if params["date"] =~ /^\d{4}/
  end

  # Format: "Author. Title. Year." or partial if fields missing
  parts = []
  parts << author unless author.empty?
  parts << "\"#{title}\"" unless title.empty?
  parts << year unless year.empty?

  parts.empty? ? "" : parts.join(". ") + "."
end

    		
  





    
      

  

  
    #format_wiki(text, config = {})  ⇒ Object 
  

  

  


  

    
      


121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167

    		
    
      
# File 'lib/wp2txt/utils.rb', line 121

def format_wiki(text, config = {})
  # Work with a mutable copy to reduce intermediate string allocations
  result = +text.to_s

  # Early exit: Skip expensive processing if no templates present
  has_templates = result.include?("{{")

  # Expand magic words if title is provided and text contains templates
  # This converts {{PAGENAME}}, {{CURRENTYEAR}}, {{lc:...}}, etc. to actual values
  if config[:title] && has_templates
    magic_expander = MagicWordExpander.new(
      config[:title],
      namespace: config[:namespace] || "",
      dump_date: config[:dump_date]
    )
    result = magic_expander.expand(result)
  end

  # Expand parser functions if enabled and text contains parser function syntax
  # This evaluates {{#if:...}}, {{#switch:...}}, {{#expr:...}}, etc.
  if config[:expand_templates] && has_templates && result.include?("{{#")
    parser_functions = ParserFunctions.new(
      reference_date: config[:dump_date]
    )
    result = parser_functions.evaluate(result)
  end

  # Expand common templates if enabled and text still contains templates
  # This converts {{birth date|...}}, {{convert|...}}, etc. to readable text
  if config[:expand_templates] && result.include?("{{")
    template_expander = TemplateExpander.new(
      reference_date: config[:dump_date]
    )
    result = template_expander.expand(result)
  end

  # CPU-intensive regex processing (can be parallelized with Ractor)
  result = format_wiki_regex_transform(result, config)

  # Decode HTML entities (e.g., &Oslash; → Ø)
  # This uses HTMLEntities gem - must be done outside Ractor
  result = special_chr(result)

  # Convert marker placeholders to final [MARKER] format
  result = finalize_markers(result)
  result
end

    		
  





    
      

  

  
    #format_wiki_regex_transform(text, config = {})  ⇒ Object 
  

  

  


  

    
CPU-intensive regex transformations - Ractor-safe (no external gem dependencies)
This is the part that benefits from parallel processing


  





  



  

    
      


171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206

    		
    
      
# File 'lib/wp2txt/utils.rb', line 171

def format_wiki_regex_transform(text, config = {})
  result = +text.to_s

  # Determine which markers are enabled
  markers_config = config.fetch(:markers, true)
  enabled_markers = parse_markers_config(markers_config)

  # Citation extraction option
  extract_citations = config.fetch(:extract_citations, false)

  # Apply markers BEFORE other processing (to preserve content for replacement)
  markers_to_apply = extract_citations ? enabled_markers - [:references] : enabled_markers
  result = apply_markers(result, markers_to_apply)

  result = remove_complex(result)
  result = escape_nowiki(result)
  result = process_interwiki_links(result)
  result = process_external_links(result)
  result = unescape_nowiki(result)

  # Use in-place modifications for simple regex replacements
  result.gsub!(REMOVE_DIRECTIVES_REGEX, "")
  result.gsub!(REMOVE_EMPHASIS_REGEX) { $2 }
  result.gsub!(MNDASH_REGEX, "–")
  result.gsub!(REMOVE_HR_REGEX, "")
  result.gsub!(REMOVE_TAG_REGEX, "")

  # Remove [ref]...[/ref] markers unless --ref option is enabled
  result = remove_ref(result) unless config[:ref]

  result = correct_inline_template(result, enabled_markers, extract_citations) unless config[:inline]
  result = remove_templates(result) unless config[:inline]
  result = remove_table(result, enabled_markers) unless config[:table]

  result
end

    		
  





    
      

  

  
    #html_decoder  ⇒ Object 
  

  

  


  

    
Get HTML decoder instance (thread-local for Ractor compatibility)


  





  



  

    
      


27
28
29

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 27

def html_decoder
  Thread.current[:wp2txt_html_decoder] ||= HTMLEntities.new
end

    		
  





    
      

  

  
    #make_reference(str)  ⇒ Object 
  

  

  


  

    
      


306
307
308
309
310
311
312
313
314

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 306

def make_reference(str)
  # Work with a mutable copy to reduce intermediate string allocations
  result = +str.to_s
  result.gsub!(MAKE_REFERENCE_REGEX_A, "\n")
  result.gsub!(MAKE_REFERENCE_REGEX_B, "")
  result.gsub!(MAKE_REFERENCE_REGEX_C, "[ref]")
  result.gsub!(MAKE_REFERENCE_REGEX_D, "[/ref]")
  result
end

    		
  





    
      

  

  
    #marker_placeholder(type)  ⇒ Object 
  

  

  


  

    
Placeholder format for markers (to avoid conflicts with bracket processing)
These get converted to [MARKER] at the end of format_wiki


  





  



  

    
      


227
228
229

    		
    
      
# File 'lib/wp2txt/utils.rb', line 227

def marker_placeholder(type)
  "\u00AB\u00AB#{type.to_s.upcase}\u00BB\u00BB"  # «« MARKER »»
end

    		
  





    
      

  

  
    #mndash(str)  ⇒ Object 
  

  

  


  

    
      


64
65
66

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 64

def mndash(str)
  str.gsub(MNDASH_REGEX, "–")
end

    		
  





    
      

  

  
    #parse_markers_config(config)  ⇒ Object 
  

  

  


  

    
Parse markers configuration
true or nil: all markers enabled
false: no markers
Array: only specified markers


  





  



  

    
      


212
213
214
215
216
217
218
219
220
221
222
223

    		
    
      
# File 'lib/wp2txt/utils.rb', line 212

def parse_markers_config(config)
  case config
  when true, nil
    DEFAULT_MARKERS.dup
  when false
    []
  when Array
    config.map(&:to_sym) & MARKER_TYPES
  else
    DEFAULT_MARKERS.dup
  end
end

    		
  





    
      

  

  
    #process_external_links(str)  ⇒ Object 
  

  

  


  

    
      


418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435

    		
    
      
# File 'lib/wp2txt/utils.rb', line 418

def process_external_links(str)
  # Early exit if no external links present
  return str unless str.include?("[")

  process_nested_single_pass(str, "[", "]") do |contents|
    if /\A\s.+\s\z/ =~ contents
      " (#{contents.strip}) "
    else
      parts = contents.split(" ", 2)
      case parts.size
      when 1
        parts.first || ""
      else
        parts.last || ""
      end
    end
  end
end

    		
  





    
      

  

  
    #process_interwiki_links(str)  ⇒ Object 
  

  

  


  

    
File/Image namespace and parameter regexes are now defined in regex.rb
FILE_NAMESPACES_REGEX - matches file namespace prefixes (313 aliases from 350+ languages)
IMAGE_PARAMS_REGEX - matches image parameters like thumb, right, left, etc.


  





  



  

    
      


365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402

    		
    
      
# File 'lib/wp2txt/utils.rb', line 365

def process_interwiki_links(str)
  # Early exit if no links present
  return str unless str.include?("[[")

  process_nested_single_pass(str, "[[", "]]") do |contents|
    # Use -1 to preserve trailing empty strings (for pipe trick detection)
    parts = contents.split("|", -1)
    first_part = parts.first || ""

    # Category links should be removed entirely (categories are extracted separately)
    if CATEGORY_NAMESPACE_REGEX.match?(first_part)
      ""
    elsif FILE_NAMESPACES_REGEX.match?(first_part)
      # For File/Image links, extract caption (last non-parameter part)
      # Normalize newlines to pipes (handles malformed markup with newlines instead of pipes)
      normalized = contents.gsub(/\n/, "|")
      parts = normalized.split("|", -1)
      # Skip parts that look like parameters (contain =, or are size specs like 200px)
      if parts.size > 1
        caption = parts[1..].reverse.find do |p|
          stripped = p.strip
          !stripped.empty? && !stripped.include?("=") && !stripped.match?(/\A\d+px\z/i) && !(IMAGE_PARAMS_REGEX && IMAGE_PARAMS_REGEX.match?(stripped))
        end
        caption&.strip || ""
      else
        ""
      end
    elsif parts.size == 1
      first_part
    elsif parts.size == 2 && parts[1].strip.empty?
      # Pipe trick: [[Namespace:Page|]] or [[Page (disambiguation)|]]
      apply_pipe_trick(first_part)
    else
      parts.shift
      parts.join("|")
    end
  end
end

    		
  





    
      

  

  
    #process_nested_single_pass(str, left, right, &block)  ⇒ Object 
  

  

  


  

    
Single-pass iterative processor - finds and processes innermost brackets first
This avoids the overhead of recursive calls and repeated string scanning


  





  



  

    
      


79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 79

def process_nested_single_pass(str, left, right, &block)
  return str unless str.include?(left)

  result = +str
  left_len = left.length
  right_len = right.length
  max_iterations = MAX_NESTING_ITERATIONS

  iterations = 0
  loop do
    iterations += 1
    break if iterations > max_iterations

    pos = 0
    found = false

    while pos < result.length
      # Find next left bracket
      left_pos = result.index(left, pos)
      break unless left_pos

      # Look for nested left bracket and matching right bracket
      inner_left = result.index(left, left_pos + left_len)
      right_pos = result.index(right, left_pos + left_len)

      break unless right_pos

      # If there's a nested left bracket before the right, skip to process inner first
      if inner_left && inner_left < right_pos
        pos = inner_left
        next
      end

      # Found innermost pair - process it
      content = result[(left_pos + left_len)...right_pos]
      processed = yield content
      result = result[0...left_pos] + processed + result[(right_pos + right_len)..]
      found = true
      break
    end

    break unless found
  end

  result
rescue RegexpError, ArgumentError, SystemStackError
  # RegexpError: malformed pattern, ArgumentError: invalid argument
  # SystemStackError: stack overflow from deeply nested content
  str
end

    		
  





    
      

  

  
    #process_nested_structure(scanner, left, right, &block)  ⇒ Object 
  

  

  


  

    
Optimized single-pass nested structure processor
Processes innermost brackets first, avoiding recursion overhead


  





  



  

    
      


72
73
74
75

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 72

def process_nested_structure(scanner, left, right, &block)
  str = scanner.is_a?(StringScanner) ? scanner.string : scanner.to_s
  process_nested_single_pass(str, left, right, &block)
end

    		
  





    
      

  

  
    #remove_complex(str)  ⇒ Object 
  

  

  


  

    
      


263
264
265
266
267
268
269
270
271
272

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 263

def remove_complex(str)
  # Work with a mutable copy to reduce intermediate string allocations
  result = +str.to_s
  result.gsub!(COMPLEX_REGEX_01) { "《#{$1}》" }
  result.gsub!(COMPLEX_REGEX_02, "")
  result.gsub!(COMPLEX_REGEX_03, "")
  result.gsub!(COMPLEX_REGEX_04, "")
  result.gsub!(COMPLEX_REGEX_05, "")
  result
end

    		
  





    
      

  

  
    #remove_directive(str)  ⇒ Object 
  

  

  


  

    
      


288
289
290

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 288

def remove_directive(str)
  str.gsub(REMOVE_DIRECTIVES_REGEX, "")
end

    		
  





    
      

  

  
    #remove_emphasis(str)  ⇒ Object 
  

  

  


  

    
      


292
293
294
295
296

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 292

def remove_emphasis(str)
  str.gsub(REMOVE_EMPHASIS_REGEX) do
    $2
  end
end

    		
  





    
      

  

  
    #remove_hr(str)  ⇒ Object 
  

  

  


  

    
      


298
299
300

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 298

def remove_hr(str)
  str.gsub(REMOVE_HR_REGEX, "")
end

    		
  





    
      

  

  
    #remove_html(str)  ⇒ Object 
  

  

  


  

    
      


239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 239

def remove_html(str)
  res = +str.to_s
  # Remove HTML comments first (before other processing to avoid [ref] in comments issue)
  res.gsub!(HTML_COMMENT_REGEX, "")
  res.gsub!(SELF_CLOSING_TAG_REGEX, "")

  # Use data-driven extension tags, filtered to block-level only
  # Combine BLOCK_EXTENSION_TAGS with extension_tags from data for comprehensive coverage
  tags_to_remove = (BLOCK_EXTENSION_TAGS + EXTENSION_TAGS.select { |t|
    # Include additional block-level tags from data
    %w[div gallery timeline noinclude imagemap poem hiero graph categorytree section abschnitt].include?(t)
  }).uniq

  tags_to_remove.each do |tag|
    # Early exit if tag not present
    next unless res.include?("<#{tag}")
    result = process_nested_single_pass(res, "<#{tag}", "#{tag}>") { "" }
    res.replace(result)
  end
  # Remove imagemap coordinate remnants (rect, poly, circle, default with coordinates)
  res.gsub!(IMAGEMAP_COORD_REGEX, "")
  res
end

    		
  





    
      

  

  
    #remove_inbetween(str, tagset = ["<", ">"])  ⇒ Object 
  

  

  


  

    
      


274
275
276
277
278
279
280
281
282

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 274

def remove_inbetween(str, tagset = ["<", ">"])
  # Use cached regex for common tagsets
  cache_key = "inbetween:#{tagset.join}"
  regex = Wp2txt.regex_cache[cache_key] ||= begin
    tagsets = Regexp.quote(tagset.uniq.join(""))
    Regexp.new("#{Regexp.escape(tagset[0])}[^#{tagsets}]*#{Regexp.escape(tagset[1])}")
  end
  str.gsub(regex, "")
end

    		
  





    
      

  

  
    #remove_ref(str)  ⇒ Object 
  

  

  


  

    
      


302
303
304

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 302

def remove_ref(str)
  str.gsub(FORMAT_REF_REGEX) { "" }
end

    		
  





    
      

  

  
    #remove_table(str, enabled_markers = [])  ⇒ Object 
  

  

  


  

    
      


450
451
452
453
454
455
456
457
458
459
460
461

    		
    
      
# File 'lib/wp2txt/utils.rb', line 450

def remove_table(str, enabled_markers = [])
  # Early exit if no tables present
  return str unless str.include?("{|")

  # If table marker is enabled, tables are already replaced with [TABLE]
  # Only remove if marker is not enabled
  if enabled_markers.include?(:table)
    str
  else
    process_nested_single_pass(str, "{|", "|}") { "" }
  end
end

    		
  





    
      

  

  
    #remove_tag(str)  ⇒ Object 
  

  

  


  

    
      


284
285
286

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 284

def remove_tag(str)
  str.gsub(REMOVE_TAG_REGEX, "")
end

    		
  





    
      

  

  
    #remove_templates(str)  ⇒ Object 
  

  

  


  

    
template processing ####################


  





  



  

    
      


439
440
441
442
443
444
445
446
447
448

    		
    
      
# File 'lib/wp2txt/utils.rb', line 439

def remove_templates(str)
  # Early exit if no templates present
  return str unless str.include?("{{")

  result = process_nested_single_pass(str, "{{", "}}") { "" }

  # Handle single brace templates (less common)
  return result unless result.include?("{")
  process_nested_single_pass(result, "{", "}") { "" }
end

    		
  





    
      

  

  
    #rename(files, ext = "txt")  ⇒ Object 
  

  

  


  

    
      


65
66
67
68
69
70
71
72
73
74
75
76
77
78

    		
    
      
# File 'lib/wp2txt/file_utils.rb', line 65

def rename(files, ext = "txt")
  # num of digits necessary to name the last file generated
  maxwidth = 0

  files.each do |f|
    width = f.slice(/-(\d+)\z/, 1).to_s.length.to_i
    maxwidth = width if maxwidth < width
    newname = f.sub(/-(\d+)\z/) do
      "-" + format("%0#{maxwidth}d", $1.to_i)
    end
    File.rename(f, newname + ".#{ext}")
  end
  true
end

    		
  





    
      

  

  
    #replace_paired_templates_with_marker(str, start_pattern, end_name, placeholder, should_mark)  ⇒ Object 
  

  

  


  

    
Replace paired templates like {refbegin}...{refend} with marker
When should_mark is false, skip processing entirely (don't remove content)
This allows extract_citations to process the inner templates


  





  



  

    
      


286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307

    		
    
      
# File 'lib/wp2txt/utils.rb', line 286

def replace_paired_templates_with_marker(str, start_pattern, end_name, placeholder, should_mark)
  return str unless should_mark  # Skip if not marking - let content be processed later

  result = +str.to_s
  end_regex = /\{\{#{Regexp.escape(end_name)}\s*\}\}/i

  loop do
    match = result.match(start_pattern)
    break unless match

    start_pos = match.begin(0)

    # Find the closing template (e.g., {{refend}})
    end_match = result.match(end_regex, start_pos)
    break unless end_match

    end_pos = end_match.end(0)

    result = result[0...start_pos] + placeholder + result[end_pos..]
  end
  result
end

    		
  





    
      

  

  
    #replace_template_with_marker(str, pattern, placeholder, should_mark)  ⇒ Object 
  

  

  


  

    
Replace templates matching pattern with marker (handles nested braces)


  





  



  

    
      


310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351

    		
    
      
# File 'lib/wp2txt/utils.rb', line 310

def replace_template_with_marker(str, pattern, placeholder, should_mark)
  result = +str.to_s
  # Find all positions where template pattern matches
  loop do
    match = result.match(pattern)
    break unless match

    start_pos = match.begin(0)
    # Find the end of this template by counting braces
    depth = 0
    pos = start_pos
    template_end = nil

    while pos < result.length
      if result[pos, 2] == "{{"
        depth += 1
        pos += 2
      elsif result[pos, 2] == "}}"
        depth -= 1
        pos += 2
        if depth == 0
          template_end = pos
          break
        end
      else
        pos += 1
      end
    end

    if template_end
      if should_mark
        result = result[0...start_pos] + placeholder + result[template_end..]
      else
        result = result[0...start_pos] + result[template_end..]
      end
    else
      # Unclosed template, break to avoid infinite loop
      break
    end
  end
  result
end

    		
  





    
      

  

  
    #replace_wiki_table_with_marker(str, placeholder)  ⇒ Object 
  

  

  


  

    
Replace wiki tables {|...|} with marker


  





  



  

    
      


354
355
356
357

    		
    
      
# File 'lib/wp2txt/utils.rb', line 354

def replace_wiki_table_with_marker(str, placeholder)
  return str unless str.include?("{|")
  process_nested_single_pass(str, "{|", "|}") { placeholder }
end

    		
  





    
      

  

  
    #sec_to_str(int)  ⇒ Object 
  

  

  


  

    
Convert int of seconds to string in the format 00:00:00


  





  



  

    
      


81
82
83
84
85
86
87
88
89
90

    		
    
      
# File 'lib/wp2txt/file_utils.rb', line 81

def sec_to_str(int)
  unless int
    str = "--:--:--"
    return str
  end
  h = int / 3600
  m = (int - h * 3600) / 60
  s = int % 60
  format("%02d:%02d:%02d", h, m, s)
end

    		
  





    
      

  

  
    #special_chr(str)  ⇒ Object 
  

  

  


  

    
      


31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 31

def special_chr(str)
  result = html_decoder.decode(str)
  # Decode additional mathematical entities not covered by HTMLEntities gem
  result.gsub!(MATH_ENTITIES_REGEX) { MATH_ENTITIES[$1] }
  result
rescue RangeError
  # RangeError: character code out of range (e.g., invalid numeric entity like &#1550315;)
  # Remove invalid numeric entities and try again
  cleaned = str.gsub(/&#(\d+);/) do |match|
    codepoint = $1.to_i
    codepoint <= 0x10FFFF ? match : ""
  end
  cleaned.gsub(/&#x([0-9a-fA-F]+);/) do |match|
    codepoint = $1.to_i(16)
    codepoint <= 0x10FFFF ? match : ""
  end
end

    		
  





    
      

  

  
    #template_matches?(name, template_list)  ⇒ Boolean 
  

  

  


  

    
Helper to check if template name matches any in a list (case-insensitive)


  





  

Returns:


    
  
    
  • 
      
      
        (Boolean)
      
      
      
    
    • 
  





  

    
      


559
560
561
562
563

    		
    
      
# File 'lib/wp2txt/utils.rb', line 559

def template_matches?(name, template_list)
  return false if template_list.nil? || template_list.empty?
  normalized_name = name.to_s.strip.downcase
  template_list.any? { |t| t.downcase == normalized_name }
end

    		
  





    
      

  

  
    #unescape_nowiki(str)  ⇒ Object 
  

  

  


  

    
      


146
147
148
149
150
151

    		
    
      
# File 'lib/wp2txt/text_processing.rb', line 146

def unescape_nowiki(str)
  str.gsub(UNESCAPE_NOWIKI_REGEX) do
    obj_id = $1.to_i
    @nowikis[obj_id]
  end
end