Class: Wp2txt::Article

Inherits:
Object
  • Object
show all
Includes:
Wp2txt
Defined in:
lib/wp2txt/article.rb

Overview

an article contains elements, each of which is [TYPE, string]

Constant Summary

Constants included from Wp2txt

AUTHORITY_CONTROL_REGEX, AUTHORITY_CONTROL_TEMPLATES, BLANK_LINE_REGEX, BLOCK_EXTENSION_TAGS, BLOCK_MARKERS, BYTES_PER_GB, BYTES_PER_KB, BYTES_PER_MB, CATEGORY_LINE_REGEX, CATEGORY_NAMESPACES, CATEGORY_NAMESPACE_REGEX, CATEGORY_REGEX, CHRREF_TO_UTF_REGEX, CITATION_TEMPLATES, CITATION_TEMPLATE_REGEX, CLEANUP_EMPTY_PARENS_REGEX, CLEANUP_FILE_INCOMPLETE_REGEX, CLEANUP_FILE_LINE_REGEX, CLEANUP_KEY_VALUE_LINE_REGEX, CLEANUP_MIXED_WHITESPACE_REGEX, CLEANUP_MULTIPLE_PIPES_REGEX, CLEANUP_MULTIPLE_SPACES_REGEX, CLEANUP_MULTI_BLANK_REGEX, CLEANUP_ORPHANED_BRACKETS_REGEX, CLEANUP_ORPHANED_CLOSE_REGEX, CLEANUP_ORPHANED_OPEN_REGEX, CLEANUP_PIPE_CLOSE_REGEX, CLEANUP_PIPE_LINE_REGEX, CLEANUP_REGEX_01, CLEANUP_REGEX_02, CLEANUP_REGEX_03, CLEANUP_REGEX_04, CLEANUP_REGEX_05, CLEANUP_REGEX_06, CLEANUP_REGEX_07, CLEANUP_REGEX_08, CLEANUP_REMNANTS_REGEX, CLEANUP_REMNANTS_TEMPLATES, CLEANUP_STANDALONE_CLOSE_REGEX, CLEANUP_TRAILING_PIPE_REGEX, COMPLEX_REGEX_01, COMPLEX_REGEX_02, COMPLEX_REGEX_03, COMPLEX_REGEX_04, COMPLEX_REGEX_05, CONVERT_TEMPLATES, COUNTRY_CODE_REGEX, CURLY_SQUARE_BRACKET_REGEX, DEFAULTSORT_KEYWORDS, DEFAULT_BUFFER_SIZE, DEFAULT_CATEGORY_CACHE_EXPIRY_DAYS, DEFAULT_DUMP_EXPIRY_DAYS, DEFAULT_HTTP_TIMEOUT, DEFAULT_MARKERS, DEFAULT_PROGRESS_INTERVAL, DEFAULT_TOP_N_SECTIONS, DEF_MARKS_REGEX, DISPLAYTITLE_KEYWORDS, DOUBLE_CURLY_BRACKET_REGEX, DOUBLE_SQUARE_BRACKET_REGEX, DOUBLE_UNDERSCORE_MAGIC_REGEX, DOUBLE_UNDERSCORE_PATTERNS, ENTITIES, ESCAPE_NOWIKI_REGEX, EXTENSION_TAGS, EXTRA_ENTITIES, EXTRA_ENTITIES_REGEX, FILE_NAMESPACES, FILE_NAMESPACES_REGEX, FLAG_TEMPLATES, FLAG_TEMPLATE_REGEX, FORMATTING_TEMPLATES, FORMATTING_TEMPLATE_REGEX, FORMAT_REF_REGEX, HEADING_CODA_REGEX, HEADING_ONSET_REGEX, HTML_COMMENT_REGEX, HTML_DECODER, HTML_ENTITIES_PATH, HTML_HASH, HTML_REGEX, IMAGEMAP_COORD_REGEX, IMAGE_PARAMS, IMAGE_PARAMS_REGEX, IMAGE_PARAM_KEYS, INDEX_PROGRESS_THRESHOLD, INLINE_MARKERS, INTERWIKI_LINK_TEMPLATES, INTERWIKI_PREFIX_REGEX, IN_DEFINITION_REGEX, IN_HEADING_REGEX, IN_HTML_TABLE_REGEX, IN_HTML_TABLE_REGEX1, IN_HTML_TABLE_REGEX2, IN_INPUTBOX_REGEX, IN_INPUTBOX_REGEX1, IN_INPUTBOX_REGEX2, IN_LINK_REGEX, IN_MATH_REGEX, IN_MATH_REGEX1, IN_MATH_REGEX2, IN_ORDERED_REGEX, IN_PRE_REGEX, IN_SOURCE_REGEX, IN_SOURCE_REGEX1, IN_SOURCE_REGEX2, IN_TABLE_REGEX1, IN_TABLE_REGEX2, IN_UNORDERED_REGEX, ISOLATED_TAG_REGEX, ISOLATED_TEMPLATE_REGEX, LIST_MARKS_REGEX, LONE_ASTERISK_REGEX, MAGIC_WORD_LINE_REGEX, MAGIC_WORD_TEMPLATE_REGEX, MAKE_REFERENCE_REGEX_A, MAKE_REFERENCE_REGEX_B, MAKE_REFERENCE_REGEX_C, MAKE_REFERENCE_REGEX_D, MARKER_PATTERNS, MARKER_TYPES, MATH_ENTITIES, MATH_ENTITIES_REGEX, MAX_BUFFER_SIZE, MAX_HTTP_RETRIES, MAX_NESTING_ITERATIONS, MEDIAWIKI_DATA_PATH, MIN_BUFFER_SIZE, MIXED_SCRIPT_TEMPLATES, ML_LINK_END_REGEX, ML_LINK_ONSET_REGEX, ML_TEMPLATE_END_REGEX, ML_TEMPLATE_ONSET_REGEX, MNDASH_REGEX, NON_ARTICLE_NAMESPACES, NON_ARTICLE_NAMESPACE_REGEX, ONSET_BAR_REGEX, PRE_MARKS_REGEX, RACTOR_SHAREABLE_EXCLUDES, REDIRECT_KEYWORDS, REDIRECT_REGEX, REMOVE_DIRECTIVES_REGEX, REMOVE_DIRECTIVES_REGEX_GENERIC, REMOVE_EMPHASIS_REGEX, REMOVE_HR_REGEX, REMOVE_INLINE_REGEX, REMOVE_ISOLATED_REGEX, REMOVE_TAG_REGEX, REMOVE_TEMPLATES, REMOVE_TEMPLATES_REGEX, RESUME_METADATA_MAX_AGE_DAYS, RUBY_TEXT_TEMPLATES, SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, SELF_CLOSING_TAG_REGEX, SINGLE_CURLY_BRACKET_REGEX, SINGLE_SQUARE_BRACKET_REGEX, SISTER_PROJECTS, TEMPLATE_DATA_PATH, UNESCAPE_NOWIKI_REGEX, VERSION, WIKIMEDIA_PROJECT_NAMES, WIKIMEDIA_PROJECT_REGEX, WIKIPEDIA_ENTITIES_PATH

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Wp2txt

#apply_markers, #apply_pipe_trick, article_page?, #batch_file_mod, build_extra_entities_regex, build_template_regex, #chrref_to_utf, #cleanup, #collect_files, #convert_characters, #correct_inline_template, #correct_separator, days_to_seconds, #escape_nowiki, #extract_template_content, file_age_days, file_fresh?, #file_mod, #finalize_markers, #format_citation, format_file_size, #format_wiki, #format_wiki_regex_transform, #html_decoder, load_html_entities, load_mediawiki_data, load_template_data, make_constants_ractor_shareable!, #make_reference, #marker_placeholder, #mndash, #parse_markers_config, #process_external_links, #process_interwiki_links, #process_nested_single_pass, #process_nested_structure, #remove_complex, #remove_directive, #remove_emphasis, #remove_hr, #remove_html, #remove_inbetween, #remove_ref, #remove_table, #remove_tag, #remove_templates, #rename, #replace_paired_templates_with_marker, #replace_template_with_marker, #replace_wiki_table_with_marker, #sec_to_str, #special_chr, ssl_safe_get, #template_matches?, #unescape_nowiki

Constructor Details

#initialize(text, title = "", strip_tmarker = false) ⇒ Article

Returns a new instance of Article.



31
32
33
34
35
36
37
38
39
 
# File 'lib/wp2txt/article.rb', line 31

def initialize(text, title = "", strip_tmarker = false)
  @title = title.strip
  @strip_tmarker = strip_tmarker
  text = convert_characters(text)
  text = text.gsub(/\|\n\n+/m) { "|\n" }
  text = remove_html(text)
  text = make_reference(text)
  parse text
end

Instance Attribute Details

#categories ⇒ Object

Returns the value of attribute categories.



29
30
31
 
# File 'lib/wp2txt/article.rb', line 29

def categories
  @categories
end

#elements ⇒ Object

Returns the value of attribute elements.



29
30
31
 
# File 'lib/wp2txt/article.rb', line 29

def elements
  @elements
end

#title ⇒ Object

Returns the value of attribute title.



29
30
31
 
# File 'lib/wp2txt/article.rb', line 29

def title
  @title
end

Instance Method Details

#create_element(tpx, text) ⇒ Object 



41
42
43
 
# File 'lib/wp2txt/article.rb', line 41

def create_element(tpx, text)
  [tpx, text]
end

#create_heading_element(text, level) ⇒ Array

Create a heading element with level information

Parameters:

  • text (String) —

    The heading text (with or without = markers)

  • level (Integer) —

    The heading level (2 for ==, 3 for ===, etc.)

Returns:

  • (Array) —

    [:mw_heading, text, level]



49
50
51
 
# File 'lib/wp2txt/article.rb', line 49

def create_heading_element(text, level)
  [:mw_heading, text, level]
end

#extract_heading_level(line) ⇒ Integer

Extract heading level from line with = markers

Parameters:

  • line (String) —

    The heading line (e.g., "== Heading ==")

Returns:

  • (Integer) —

    The heading level (count of = signs)



56
57
58
59
 
# File 'lib/wp2txt/article.rb', line 56

def extract_heading_level(line)
  match = line.match(/^(=+)/)
  match ? match[1].length : 2
end

#extract_heading_text(line) ⇒ String

Extract clean heading text without = markers

Parameters:

  • line (String) —

    The heading line

Returns:

  • (String) —

    The heading text without = markers



64
65
66
 
# File 'lib/wp2txt/article.rb', line 64

def extract_heading_text(line)
  line.gsub(/^=+\s*/, "").gsub(/\s*=+$/, "").strip
end

#has_unbalanced_link_brackets?(line) ⇒ Boolean

Check if a line has unbalanced [[ ]] brackets Returns true if there are more [[ than ]] (indicating multi-line link)

Returns:

  • (Boolean) 


70
71
72
73
74
 
# File 'lib/wp2txt/article.rb', line 70

def has_unbalanced_link_brackets?(line)
  open_count = line.scan(/\[\[/).size
  close_count = line.scan(/\]\]/).size
  open_count > close_count
end

#parse(source) ⇒ Object 



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
 
# File 'lib/wp2txt/article.rb', line 114

def parse(source)
  @elements = []
  @categories = []
  mode = nil
  @brace_depth = 0
  source.each_line do |line|
    # Collect categories without deduplicating on each line (O(n²) → O(n))
    matched = line.scan(CATEGORY_REGEX)
    @categories.concat(matched) if matched && !matched.empty?

    case mode
    when :mw_ml_template
      # Track brace depth to find where template actually ends
      remaining = process_ml_template_line(line)
      if remaining
        # Template closed, remaining content needs to be processed
        mode = nil
        # Process remaining content if any
        unless remaining.strip.empty?
          @elements << create_element(:mw_paragraph, "\n" + remaining)
        end
      end
      next
    when :mw_ml_link
      scanner = StringScanner.new(line)
      str = process_nested_structure(scanner, "[[", "]]") { "" }
      mode = nil if ML_LINK_END_REGEX =~ str
      @elements.last.last << line
      next
    when :mw_table
      mode = nil if IN_TABLE_REGEX2 =~ line
      @elements.last.last << line
      next
    when :mw_inputbox
      mode = nil if IN_INPUTBOX_REGEX2 =~ line
      @elements.last.last << line
      next
    when :mw_source
      mode = nil if IN_SOURCE_REGEX2 =~ line
      @elements.last.last << line
      next
    when :mw_math
      mode = nil if IN_MATH_REGEX2 =~ line
      @elements.last.last << line
      next
    when :mw_htable
      mode = nil if IN_HTML_TABLE_REGEX2 =~ line
      @elements.last.last << line
      next
    end

    case line
    when ISOLATED_TEMPLATE_REGEX
      @elements << create_element(:mw_isolated_template, line)
    when ISOLATED_TAG_REGEX
      @elements << create_element(:mw_isolated_tag, line)
    when BLANK_LINE_REGEX
      @elements << create_element(:mw_blank, "\n")
    when REDIRECT_REGEX
      @elements << create_element(:mw_redirect, line)
    when IN_HEADING_REGEX
      level = extract_heading_level(line)
      # Keep original format for backward compatibility, but also store level
      formatted_line = line.sub(HEADING_ONSET_REGEX) { $1 }.sub(HEADING_CODA_REGEX) { $1 }
      @elements << create_heading_element("\n" + formatted_line + "\n", level)
    when IN_INPUTBOX_REGEX
      @elements << create_element(:mw_inputbox, line)
    when ML_TEMPLATE_ONSET_REGEX
      @elements << create_element(:mw_ml_template, line)
      mode = :mw_ml_template
      # Count initial braces: count {{ minus }} in this line
      @brace_depth = line.scan(/\{\{/).size - line.scan(/\}\}/).size
    when ML_LINK_ONSET_REGEX
      # Only treat as multi-line link if brackets are actually unbalanced
      if has_unbalanced_link_brackets?(line)
        @elements << create_element(:mw_ml_link, line)
        mode = :mw_ml_link
      else
        # Brackets are balanced, treat as paragraph
        @elements << create_element(:mw_paragraph, "\n" + line)
      end
    when IN_INPUTBOX_REGEX1
      mode = :mw_inputbox
      @elements << create_element(:mw_inputbox, line)
    when IN_SOURCE_REGEX
      @elements << create_element(:mw_source, line)
    when IN_SOURCE_REGEX1
      mode = :mw_source
      @elements << create_element(:mw_source, line)
    when IN_MATH_REGEX
      @elements << create_element(:mw_math, line)
    when IN_MATH_REGEX1
      mode = :mw_math
      @elements << create_element(:mw_math, line)
    when IN_HTML_TABLE_REGEX
      @elements << create_element(:mw_htable, line)
    when IN_HTML_TABLE_REGEX1
      mode = :mw_htable
      @elements << create_element(:mw_htable, line)
    when IN_TABLE_REGEX1
      mode = :mw_table
      @elements << create_element(:mw_table, line)
    when IN_UNORDERED_REGEX
      line = line.sub(LIST_MARKS_REGEX, "") if @strip_tmarker
      @elements << create_element(:mw_unordered, line)
    when IN_ORDERED_REGEX
      line = line.sub(LIST_MARKS_REGEX, "") if @strip_tmarker
      @elements << create_element(:mw_ordered, line)
    when IN_PRE_REGEX
      line = line.sub(PRE_MARKS_REGEX, "") if @strip_tmarker
      @elements << create_element(:mw_pre, line)
    when IN_DEFINITION_REGEX
      line = line.sub(DEF_MARKS_REGEX, "") if @strip_tmarker
      @elements << create_element(:mw_definition, line)
    when IN_LINK_REGEX
      @elements << create_element(:mw_link, line)
    else
      @elements << create_element(:mw_paragraph, "\n" + line)
    end
  end
  # Deduplicate categories once at the end (O(n) instead of O(n²))
  @categories.uniq!
  @elements
end

#process_ml_template_line(line) ⇒ Object

Process a line in multi-line template mode, tracking brace depth Updates @brace_depth and returns remaining content after }} if template closed, nil otherwise



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/wp2txt/article.rb', line 78

def process_ml_template_line(line)
  pos = 0
  close_pos = nil

  while pos < line.length
    open_idx = line.index("{{", pos)
    close_idx = line.index("}}", pos)

    if open_idx && (!close_idx || open_idx < close_idx)
      @brace_depth += 1
      pos = open_idx + 2
    elsif close_idx
      @brace_depth -= 1
      pos = close_idx + 2
      if @brace_depth == 0
        close_pos = close_idx + 2
        break
      end
    else
      break
    end
  end

  if close_pos
    # Template closed at close_pos
    template_part = line[0...close_pos]
    remaining = line[close_pos..]
    @elements.last.last << template_part
    remaining
  else
    # Template continues
    @elements.last.last << line
    nil
  end
end