Class: Wp2txt::SectionExtractor

Inherits:
Object
  • Object
show all
Defined in:
lib/wp2txt/section_extractor.rb

Overview

SectionExtractor handles extraction of sections from Wikipedia articles Supports both metadata extraction (headings only) and content extraction

Constant Summary collapse

SUMMARY_KEY =

Reserved keyword for the lead section (text before first heading)

"summary"
DEFAULT_ALIASES =

Default section aliases (canonical name => array of aliases) These cover common variations found across English Wikipedia articles. Users can add custom aliases via --alias-file for other languages or domains.

{
  "Plot" => ["Synopsis", "Plot summary", "Story"],
  "Reception" => ["Critical reception", "Reviews", "Critical response"],
  "References" => ["Notes", "Footnotes", "Citations", "Notes and references"],
  "External links" => ["External sources"],
  "See also" => ["Related articles", "Related pages"],
  "Bibliography" => ["Works", "Publications", "Selected works", "Selected bibliography"],
  "Awards" => ["Awards and nominations", "Honors", "Accolades"],
  "Legacy" => ["Impact", "Influence", "Cultural impact", "Cultural legacy"],
  "Early life" => ["Early life and education", "Childhood", "Early years"],
  "Career" => ["Professional career"],
  "Filmography" => ["Films"],
  "Discography" => ["Discography and videography"]
}.freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(target_sections = nil, options = {}) ⇒ SectionExtractor

Returns a new instance of SectionExtractor.

Parameters:

  • target_sections (Array<String>, nil) (defaults to: nil) —

    List of section names to extract (nil = all)

  • options (Hash) (defaults to: {}) —

    Extraction options

Options Hash (options):

  • :min_length (Integer) —

    Minimum section length (default: 0)

  • :skip_empty (Boolean) —

    Skip articles with no matching sections (default: false)

  • :aliases (Hash) —

    Custom section aliases (merged with defaults)

  • :alias_file (String) —

    Path to YAML file with custom aliases

  • :use_aliases (Boolean) —

    Enable alias matching (default: true)

  • :track_matches (Boolean) —

    Track which headings matched (default: false)



42
43
44
45
46
47
48
49
50
 
# File 'lib/wp2txt/section_extractor.rb', line 42

def initialize(target_sections = nil, options = {})
  @targets = normalize_targets(target_sections)
  @min_length = options[:min_length] || 0
  @skip_empty = options[:skip_empty] || false
  @use_aliases = options.fetch(:use_aliases, true)
  @track_matches = options[:track_matches] || false
  @matched_sections = {}
  @aliases = build_aliases(options[:aliases], options[:alias_file])
end

Instance Attribute Details

#matched_sections ⇒ Object (readonly)

Track which actual headings matched which requested sections



32
33
34
 
# File 'lib/wp2txt/section_extractor.rb', line 32

def matched_sections
  @matched_sections
end

Class Method Details

.load_aliases_from_file(file_path) ⇒ Hash

Load aliases from YAML file

Parameters:

  • file_path (String) —

    Path to YAML file

Returns:

  • (Hash) —

    Aliases hash (canonical => [aliases])



55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/wp2txt/section_extractor.rb', line 55

def self.load_aliases_from_file(file_path)
  return {} unless file_path && File.exist?(file_path)

  data = YAML.safe_load(File.read(file_path), permitted_classes: [Symbol])
  return {} unless data.is_a?(Hash)

  # Normalize: ensure values are arrays
  data.transform_values { |v| Array(v) }
rescue Psych::SyntaxError, Errno::ENOENT
  {}
end

Instance Method Details

#extract_headings(article) ⇒ Array<String>

Extract section headings from article (for --metadata-only)

Parameters:

  • article (Article) —

    The article to extract from

Returns:

  • (Array<String>) —

    List of section heading names



70
71
72
73
74
75
76
77
78
79
 
# File 'lib/wp2txt/section_extractor.rb', line 70

def extract_headings(article)
  headings = []
  article.elements.each do |element|
    next unless element[0] == :mw_heading

    heading_text = clean_heading_text(element[1])
    headings << heading_text unless heading_text.empty?
  end
  headings
end

#extract_headings_with_levels(article) ⇒ Array<Hash>

Extract section headings with levels (for detailed analysis)

Parameters:

  • article (Article) —

    The article to extract from

Returns:

  • (Array<Hash>) —

    List of level: hashes



84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/wp2txt/section_extractor.rb', line 84

def extract_headings_with_levels(article)
  headings = []
  article.elements.each do |element|
    next unless element[0] == :mw_heading

    heading_text = clean_heading_text(element[1])
    level = element[2] || 2
    headings << { name: heading_text, level: level } unless heading_text.empty?
  end
  headings
end

#extract_sections(article, config = {}) ⇒ Hash

Extract specified sections from article

Parameters:

  • article (Article) —

    The article to extract from

  • config (Hash) (defaults to: {}) —

    Formatting configuration

Returns:

  • (Hash) —

    Section name => content (nil if not found)



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/wp2txt/section_extractor.rb', line 121

def extract_sections(article, config = {})
  return {} if @targets.nil? || @targets.empty?

  # Reset matched sections for this article
  @matched_sections = {}

  result = {}

  # Initialize all targets with nil
  @targets.each { |t| result[t] = nil }

  # Handle summary separately
  if @targets.include?(SUMMARY_KEY)
    summary = extract_summary(article, config)
    result[SUMMARY_KEY] = apply_min_length_filter(summary)
  end

  # Extract other sections
  current_section = nil
  current_level = nil
  buffer = +""

  article.elements.each do |element|
    type = element[0]
    content = element[1]
    level = element[2]

    if type == :mw_heading
      # Save previous section if it was a target
      if current_section
        canonical = find_canonical_name(current_section)
        if canonical && canonical != SUMMARY_KEY
          result[canonical] = apply_min_length_filter(buffer.strip)
        end
      end

      # Check if this heading is a target
      heading_text = clean_heading_text(content)
      canonical = find_canonical_name(heading_text)

      if canonical && canonical != SUMMARY_KEY
        current_section = heading_text
        current_level = level || 2
        buffer = +""
      elsif current_level && (level.nil? || level <= current_level)
        # Same or higher level heading ends current section
        current_section = nil
        current_level = nil
        buffer = +""
      end
    elsif current_section
      # Accumulate content for current section
      buffer << content.to_s
    end
  end

  # Save final section
  if current_section
    canonical = find_canonical_name(current_section)
    if canonical && canonical != SUMMARY_KEY
      result[canonical] = apply_min_length_filter(buffer.strip)
    end
  end

  result
end

#extract_summary(article, config = {}) ⇒ String?

Extract summary (lead section) from article

Parameters:

  • article (Article) —

    The article to extract from

  • config (Hash) (defaults to: {}) —

    Formatting configuration

Returns:

  • (String, nil) —

    The summary text or nil if empty



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/wp2txt/section_extractor.rb', line 100

def extract_summary(article, config = {})
  contents = +""
  article.elements.each do |element|
    # Stop at first heading
    break if element[0] == :mw_heading

    # Skip non-content elements
    next if %i[mw_blank mw_redirect mw_comment].include?(element[0])

    content = element[1].to_s
    contents << content
  end

  result = contents.strip
  result.empty? ? nil : result
end

#has_matching_sections?(article) ⇒ Boolean

Check if article has any matching sections

Parameters:

  • article (Article) —

    The article to check

Returns:

  • (Boolean) —

    true if at least one target section exists



191
192
193
194
195
196
197
198
199
200
201
202
203
 
# File 'lib/wp2txt/section_extractor.rb', line 191

def has_matching_sections?(article)
  return true if @targets.nil? || @targets.empty?

  # Check summary
  if @targets.include?(SUMMARY_KEY)
    summary = extract_summary(article)
    return true if summary && !summary.empty?
  end

  # Check headings (don't record matches during check)
  headings = extract_headings(article)
  headings.any? { |h| find_canonical_name(h, record_match: false) }
end

#should_skip?(article) ⇒ Boolean

Check if extraction should be skipped for this article

Parameters:

  • article (Article) —

    The article to check

Returns:

  • (Boolean) —

    true if article should be skipped



208
209
210
211
 
# File 'lib/wp2txt/section_extractor.rb', line 208

def should_skip?(article)
  return false unless @skip_empty
  !has_matching_sections?(article)
end