Module: Wp2txt::Formatter

Defined in:
lib/wp2txt/formatter.rb

Overview

Article formatting utilities for WpApp

Instance Method Summary collapse

Instance Method Details

#build_text_content(article, config) ⇒ Object

Build text content from article elements



241
242
243
244
245
246
247
248
249
 
# File 'lib/wp2txt/formatter.rb', line 241

def build_text_content(article, config)
  contents = +""
  article.elements.each do |e|
    line = process_element(e, config)
    contents << line if line
  end
  # Apply cleanup to remove leftover markup, normalize whitespace, etc.
  cleanup(contents)
end

#extract_redirect(article) ⇒ Object

Extract redirect target from article if it's a redirect



219
220
221
222
223
224
225
226
227
 
# File 'lib/wp2txt/formatter.rb', line 219

def extract_redirect(article)
  article.elements.each do |type, content|
    if type == :mw_redirect
      match = content.match(REDIRECT_REGEX)
      return match[1] if match
    end
  end
  nil
end

#format_article(article, config) ⇒ Object

Format article based on configuration and output format



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/wp2txt/formatter.rb', line 16

def format_article(article, config)
  # Store original title for magic word expansion in content
  original_title = article.title.dup
  article.title = format_wiki(article.title, config)

  # Add title to config for magic word expansion in content processing
  config_with_title = config.merge(title: original_title)

  # Handle metadata_only mode (title + sections + categories)
  if config[:metadata_only]
    return format_metadata_only(article, config_with_title)
  end

  # Handle summary_only as section extraction (for consistency)
  if config[:summary_only]
    summary_config = config_with_title.merge(
      sections: [SectionExtractor::SUMMARY_KEY],
      section_output: "combined"
    )
    return format_with_sections(article, summary_config)
  end

  # Handle section extraction mode (--sections option)
  if config[:sections] && !config[:sections].empty?
    return format_with_sections(article, config_with_title)
  end

  if config[:format] == :json
    format_article_json(article, config_with_title)
  else
    format_article_text(article, config_with_title)
  end
end

#format_article_json(article, config) ⇒ Object

Format article as JSON hash



193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/wp2txt/formatter.rb', line 193

def format_article_json(article, config)
  result = { "title" => article.title }

  # Categories
  if config[:category]
    result["categories"] = article.categories.flatten
  else
    result["categories"] = nil
  end

  # Text content
  if config[:category_only]
    result["text"] = nil
  else
    text = build_text_content(article, config)
    result["text"] = text.strip
  end

  # Redirect
  redirect_target = extract_redirect(article)
  result["redirect"] = redirect_target

  result
end

#format_article_text(article, config) ⇒ Object

Format article as text string



230
231
232
233
234
235
236
237
238
 
# File 'lib/wp2txt/formatter.rb', line 230

def format_article_text(article, config)
  if config[:category_only]
    format_category_only(article)
  elsif config[:category] && !article.categories.empty?
    format_with_categories(article, config)
  else
    format_full_article(article, config)
  end
end

#format_category_only(article) ⇒ Object

Format article with only category information (text format)



252
253
254
255
256
257
 
# File 'lib/wp2txt/formatter.rb', line 252

def format_category_only(article)
  title = "#{article.title}\t"
  contents = article.categories.join(", ")
  contents << "\n"
  title + contents
end

#format_full_article(article, config) ⇒ Object

Format full article content



273
274
275
276
277
278
 
# File 'lib/wp2txt/formatter.rb', line 273

def format_full_article(article, config)
  title = "\n[[#{article.title}]]\n\n"
  contents = build_text_content(article, config)

  config[:title] ? title + contents : contents
end

#format_metadata_only(article, config) ⇒ Object

Format article with metadata only (title, section headings, categories) Used for analyzing section distribution across Wikipedia dumps



162
163
164
165
166
167
168
169
170
171
 
# File 'lib/wp2txt/formatter.rb', line 162

def format_metadata_only(article, config)
  extractor = SectionExtractor.new
  sections = extractor.extract_headings(article)

  if config[:format] == :json
    format_metadata_only_json(article, sections)
  else
    format_metadata_only_text(article, sections)
  end
end

#format_metadata_only_json(article, sections) ⇒ Object

Format metadata as JSON



174
175
176
177
178
179
180
 
# File 'lib/wp2txt/formatter.rb', line 174

def format_metadata_only_json(article, sections)
  {
    "title" => article.title,
    "sections" => sections,
    "categories" => article.categories.flatten
  }
end

#format_metadata_only_text(article, sections) ⇒ Object

Format metadata as TSV text Format: TitleSection1|Section2|...Category1,Category2,...



184
185
186
187
188
189
190
 
# File 'lib/wp2txt/formatter.rb', line 184

def format_metadata_only_text(article, sections)
  title = article.title
  sections_str = sections.join("|")
  categories_str = article.categories.flatten.join(",")

  "#{title}\t#{sections_str}\t#{categories_str}\n"
end

#format_sections_combined_json(article, sections, config, matched_sections = {}) ⇒ Object

Format sections as combined JSON (all sections concatenated)



105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/wp2txt/formatter.rb', line 105

def format_sections_combined_json(article, sections, config, matched_sections = {})
  included = sections.keys.select { |k| sections[k] && !sections[k].empty? }
  text = included.map { |k| sections[k] }.join("\n\n")

  result = {
    "title" => article.title,
    "text" => text,
    "sections_included" => included
  }
  result["categories"] = article.categories.flatten if config[:category]
  # Include matched_sections if tracking is enabled and there are matches
  if config[:show_matched_sections] && matched_sections && !matched_sections.empty?
    result["matched_sections"] = matched_sections
  end
  result
end

#format_sections_combined_text(article, sections, config) ⇒ Object

Format sections as combined text



143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'lib/wp2txt/formatter.rb', line 143

def format_sections_combined_text(article, sections, config)
  included = sections.keys.select { |k| sections[k] && !sections[k].empty? }
  text = included.map { |k| sections[k] }.join("\n\n")

  output = +"TITLE: #{article.title}\n"
  output << "SECTIONS: #{included.join(', ')}\n\n"
  output << text
  output << "\n\n"

  if config[:category] && !article.categories.empty?
    output << "CATEGORIES: #{article.categories.flatten.join(', ')}\n"
  end

  output << "\n"
  output
end

#format_sections_structured_json(article, sections, config, matched_sections = {}) ⇒ Object

Format sections as structured JSON (each section as separate field)



91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/wp2txt/formatter.rb', line 91

def format_sections_structured_json(article, sections, config, matched_sections = {})
  result = {
    "title" => article.title,
    "sections" => sections
  }
  result["categories"] = article.categories.flatten if config[:category]
  # Include matched_sections if tracking is enabled and there are matches
  if config[:show_matched_sections] && matched_sections && !matched_sections.empty?
    result["matched_sections"] = matched_sections
  end
  result
end

#format_sections_structured_text(article, sections, config) ⇒ Object

Format sections as structured text



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/wp2txt/formatter.rb', line 123

def format_sections_structured_text(article, sections, config)
  output = +"TITLE: #{article.title}\n\n"

  sections.each do |name, content|
    if content.nil?
      output << "SECTION [#{name}]: (not found)\n\n"
    else
      output << "SECTION [#{name}]:\n#{content}\n\n"
    end
  end

  if config[:category] && !article.categories.empty?
    output << "CATEGORIES: #{article.categories.flatten.join(', ')}\n"
  end

  output << "\n"
  output
end

#format_with_categories(article, config) ⇒ Object

Format article with categories (includes body text)



260
261
262
263
264
265
266
267
268
269
270
 
# File 'lib/wp2txt/formatter.rb', line 260

def format_with_categories(article, config)
  title = "\n[[#{article.title}]]\n\n"
  contents = build_text_content(article, config)

  # Add categories at the end
  contents << "\nCATEGORIES: "
  contents << article.categories.join(", ")
  contents << "\n\n"

  config[:title] ? title + contents : contents
end

#format_with_sections(article, config) ⇒ Object

Format article with specific section extraction



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/wp2txt/formatter.rb', line 51

def format_with_sections(article, config)
  extractor = SectionExtractor.new(
    config[:sections],
    min_length: config[:min_section_length] || 0,
    skip_empty: config[:skip_empty] || false,
    use_aliases: !config[:no_section_aliases],
    alias_file: config[:alias_file],
    track_matches: config[:show_matched_sections] || false
  )

  # Skip article if no matching sections and skip_empty is true
  return nil if extractor.should_skip?(article)

  sections = extractor.extract_sections(article, config)
  matched_sections = extractor.matched_sections

  # Apply format_wiki to section content
  sections.transform_values! do |content|
    next nil if content.nil?
    cleanup(format_wiki(content, config))
  end

  output_mode = config[:section_output] || "structured"

  if config[:format] == :json
    if output_mode == "combined"
      format_sections_combined_json(article, sections, config, matched_sections)
    else
      format_sections_structured_json(article, sections, config, matched_sections)
    end
  else
    if output_mode == "combined"
      format_sections_combined_text(article, sections, config)
    else
      format_sections_structured_text(article, sections, config)
    end
  end
end

#formatter_debug_mode ⇒ Object

Debug mode flag (inherited from including class if defined)



11
12
13
 
# File 'lib/wp2txt/formatter.rb', line 11

def formatter_debug_mode
  defined?(DEBUG_MODE) ? DEBUG_MODE : false
end

#process_element(element, config) ⇒ Object

Process individual element of the article



281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
 
# File 'lib/wp2txt/formatter.rb', line 281

def process_element(element, config)
  type, content = element
  debug_mode = formatter_debug_mode

  case type
  when :mw_heading
    return nil if config[:summary_only]
    return nil unless config[:heading]

    content = format_wiki(content, config)
    content += "+HEADING+" if debug_mode
    content + "\n"
  when :mw_paragraph
    content = format_wiki(content, config)
    content += "+PARAGRAPH+" if debug_mode
    content + "\n"
  when :mw_table, :mw_htable
    return nil unless config[:table]

    content += "+TABLE+" if debug_mode
    content + "\n"
  when :mw_pre
    return nil unless config[:pre]

    content += "+PRE+" if debug_mode
    content + "\n"
  when :mw_quote
    content += "+QUOTE+" if debug_mode
    content + "\n"
  when :mw_unordered, :mw_ordered, :mw_definition
    return nil unless config[:list]

    content += "+LIST+" if debug_mode
    content + "\n"
  when :mw_ml_template
    return nil unless config[:multiline]

    content += "+MLTEMPLATE+" if debug_mode
    content + "\n"
  when :mw_link
    content = format_wiki(content, config)
    return nil if content.strip.empty?

    content += "+LINK+" if debug_mode
    content + "\n"
  when :mw_ml_link
    content = format_wiki(content, config)
    return nil if content.strip.empty?

    content += "+MLLINK+" if debug_mode
    content + "\n"
  when :mw_redirect
    return nil unless config[:redirect]

    content += "+REDIRECT+" if debug_mode
    content + "\n\n"
  when :mw_isolated_template
    return nil unless config[:multiline]

    content += "+ISOLATED_TEMPLATE+" if debug_mode
    content + "\n"
  when :mw_isolated_tag
    nil
  else
    return nil unless debug_mode

    content += "+OTHER+"
    content + "\n"
  end
end