Class: Wp2txt::MetadataIndex

Inherits:
Object
  • Object
show all
Defined in:
lib/wp2txt/metadata_index.rb

Overview

Local metadata index (Tier 1) built from a multistream dump. Stores per-page categories, section headings, redirects, and the category hierarchy in SQLite, enabling offline exhaustive queries such as "all articles in category X that have a Plot section" without any API access.

Constant Summary collapse

SCHEMA_VERSION = 
2
CACHE_SUFFIX = 
"_meta.sqlite3"
NS_ARTICLE = 
0
NS_CATEGORY = 
14

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(db_path) ⇒ MetadataIndex

Returns a new instance of MetadataIndex.



27
28
29
30
 
# File 'lib/wp2txt/metadata_index.rb', line 27

def initialize(db_path)
  @db_path = db_path
  @db = nil
end

Instance Attribute Details

#db_path ⇒ Object (readonly)

Returns the value of attribute db_path.



25
26
27
 
# File 'lib/wp2txt/metadata_index.rb', line 25

def db_path
  @db_path
end

Class Method Details

.clean_heading(text) ⇒ Object

Remove wiki markup from a heading ('''bold''', [[link|label]], HTML tags)



50
51
52
53
54
 
# File 'lib/wp2txt/metadata_index.rb', line 50

def self.clean_heading(text)
  t = text.gsub(/'{2,}/, "")
  t = t.gsub(/\[\[(?:[^\]|]*\|)?([^\]]*)\]\]/) { ::Regexp.last_match(1) }
  t.gsub(/<[^>]+>/, "").strip
end

.expand_section_names(name, alias_file: nil) ⇒ Object

Expand a section name to its full alias group (bidirectional): "Plot" => ["Plot", "Synopsis", ...]; "Synopsis" => same group



58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/wp2txt/metadata_index.rb', line 58

def self.expand_section_names(name, alias_file: nil)
  aliases = SectionExtractor::DEFAULT_ALIASES
  if alias_file
    custom = SectionExtractor.load_aliases_from_file(alias_file)
    aliases = aliases.merge(custom) unless custom.empty?
  end

  down = name.downcase
  aliases.each do |canonical, list|
    group = [canonical, *list]
    return group if group.any? { |g| g.downcase == down }
  end
  [name]
end

.normalize_category(name) ⇒ Object

Normalize a category name the way MediaWiki treats titles: underscores to spaces, trimmed, first letter capitalized



42
43
44
45
46
47
 
# File 'lib/wp2txt/metadata_index.rb', line 42

def self.normalize_category(name)
  n = name.to_s.tr("_", " ").strip.squeeze(" ")
  return n if n.empty?

  n[0].upcase + n[1..].to_s
end

.path_for(multistream_path, cache_dir: nil) ⇒ Object

Default index location for a given multistream file (mirrors IndexCache naming)



33
34
35
36
37
38
 
# File 'lib/wp2txt/metadata_index.rb', line 33

def self.path_for(multistream_path, cache_dir: nil)
  dir = cache_dir || File.expand_path("~/.wp2txt/cache")
  basename = File.basename(multistream_path, ".*").sub(/\.xml\z/, "")
  path_hash = Digest::MD5.hexdigest(multistream_path)[0, 8]
  File.join(dir, "#{basename}_#{path_hash}#{CACHE_SUFFIX}")
end

Instance Method Details

#built? ⇒ Boolean

True if the index file exists and has a compatible schema

Returns:

  • (Boolean) 


78
79
80
81
82
83
84
85
 
# File 'lib/wp2txt/metadata_index.rb', line 78

def built?
  return false unless File.exist?(@db_path)

  meta = read_metadata
  !meta.nil? && meta[:schema_version].to_i == SCHEMA_VERSION && !meta[:built_at].nil?
rescue SQLite3::Exception
  false
end

#categories_of(title) ⇒ Array<String>?

Categories of one article (by exact title)

Returns:

  • (Array<String>, nil) —

    category names, nil if the title is unknown



263
264
265
266
267
268
269
270
 
# File 'lib/wp2txt/metadata_index.rb', line 263

def categories_of(title)
  row = open_db.execute("SELECT page_id FROM pages WHERE title = ?", [title]).first
  return nil unless row

  open_db.execute(
    "SELECT category FROM page_categories WHERE page_id = ? ORDER BY category", [row[0]]
  ).map(&:first)
end

#category_tree(category, depth: 2) ⇒ Object

Subcategory tree starting at category, as [depth:, ...] (BFS order)



273
274
275
276
277
278
279
280
281
282
283
284
285
286
 
# File 'lib/wp2txt/metadata_index.rb', line 273

def category_tree(category, depth: 2)
  cat = self.class.normalize_category(category)
  sql = <<~SQL
    WITH RECURSIVE cat_tree(name, d) AS (
      SELECT ?, 0
      UNION
      SELECT ch.child, ct.d + 1
      FROM category_hierarchy ch JOIN cat_tree ct ON ch.parent = ct.name
      WHERE ct.d < #{depth.to_i}
    )
    SELECT name, MIN(d) FROM cat_tree GROUP BY name ORDER BY MIN(d), name
  SQL
  open_db.execute(sql, [cat]).map { |name, d| { name: name, depth: d } }
end

#close ⇒ Object 



114
115
116
117
 
# File 'lib/wp2txt/metadata_index.rb', line 114

def close
  @db&.close
  @db = nil
end

#count_articles(category: nil, depth: 0, categories: nil, category_match: nil, has_section: nil, sections: nil, alias_set: nil, use_aliases: true, alias_file: nil, title_match: nil) ⇒ Object

Count articles matching the same filters as find_articles



246
247
248
249
250
251
252
253
254
255
256
257
258
259
 
# File 'lib/wp2txt/metadata_index.rb', line 246

def count_articles(category: nil, depth: 0, categories: nil, category_match: nil,
                   has_section: nil, sections: nil, alias_set: nil,
                   use_aliases: true, alias_file: nil, title_match: nil)
  cte, where, params = build_article_query(
    category: category, depth: depth, categories: categories, category_match: category_match,
    has_section: has_section, sections: sections,
    alias_set: alias_set, use_aliases: use_aliases, alias_file: alias_file, title_match: title_match
  )
  sql = +""
  sql << "WITH RECURSIVE #{cte} " if cte
  sql << "SELECT COUNT(*) FROM pages p WHERE #{where}"

  open_db.get_first_value(sql, params).to_i
end

#delete_alias_set(name) ⇒ Object 



395
396
397
398
399
 
# File 'lib/wp2txt/metadata_index.rb', line 395

def delete_alias_set(name)
  ensure_alias_table
  open_db.execute("DELETE FROM alias_sets WHERE name = ?", [name])
  nil
end

#finalize_build!(source_path) ⇒ Object 



183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/wp2txt/metadata_index.rb', line 183

def finalize_build!(source_path)
  db = open_db
  db.execute("CREATE INDEX IF NOT EXISTS idx_pages_title ON pages(title)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_pc_category ON page_categories(category)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_pc_page ON page_categories(page_id)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_ps_heading ON page_sections(heading COLLATE NOCASE)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_ps_page ON page_sections(page_id)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_ch_parent ON category_hierarchy(parent)")

  stat = File.stat(source_path)
  dump_name = File.basename(source_path)[/\A[a-z0-9_\-]+?-\d{8}/] || File.basename(source_path)
  save_metadata(
    schema_version: SCHEMA_VERSION,
    wp2txt_version: Wp2txt::VERSION,
    source_path: source_path,
    source_size: stat.size,
    source_mtime: stat.mtime.to_i,
    dump_name: dump_name,
    built_at: Time.now.utc.iso8601
  )
  db.execute("ANALYZE")
  close
  if @build_path
    File.rename(@build_path, @db_path)
    FileUtils.rm_f(["#{@db_path}-wal", "#{@db_path}-shm"])
    @build_path = nil
  end
end

#find_articles(category: nil, depth: 0, categories: nil, category_match: nil, has_section: nil, sections: nil, alias_set: nil, use_aliases: true, alias_file: nil, title_match: nil, limit: 0, offset: 0) ⇒ Array<String>

Find article titles matching the given filters.

Parameters:

  • category (String, nil) (defaults to: nil) —

    category name (without namespace prefix)

  • depth (Integer) (defaults to: 0) —

    subcategory recursion depth (0 = exact category only)

  • has_section (String, nil) (defaults to: nil) —

    single section heading (alias-aware by default)

  • sections (Array<String>, nil) (defaults to: nil) —

    multiple headings (OR match, used as-is)

  • alias_set (String, nil) (defaults to: nil) —

    saved alias set name used to expand headings

  • use_aliases (Boolean) (defaults to: true) —

    expand has_section via built-in alias groups

  • alias_file (String, nil) (defaults to: nil) —

    custom alias YAML (merged with defaults)

  • title_match (String, nil) (defaults to: nil) —

    substring match on title

  • limit (Integer) (defaults to: 0) —

    max titles to return (0 = no limit)

  • offset (Integer) (defaults to: 0) —

    result offset

Returns:

  • (Array<String>) —

    matching titles ordered by page_id



228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
 
# File 'lib/wp2txt/metadata_index.rb', line 228

def find_articles(category: nil, depth: 0, categories: nil, category_match: nil,
                  has_section: nil, sections: nil, alias_set: nil,
                  use_aliases: true, alias_file: nil, title_match: nil, limit: 0, offset: 0)
  cte, where, params = build_article_query(
    category: category, depth: depth, categories: categories, category_match: category_match,
    has_section: has_section, sections: sections,
    alias_set: alias_set, use_aliases: use_aliases, alias_file: alias_file, title_match: title_match
  )
  sql = +""
  sql << "WITH RECURSIVE #{cte} " if cte
  sql << "SELECT p.title FROM pages p WHERE #{where} ORDER BY p.page_id"
  sql << " LIMIT #{limit.to_i}" if limit.to_i.positive?
  sql << " OFFSET #{offset.to_i}" if offset.to_i.positive?

  open_db.execute(sql, params).map { |row| row[0] }
end

#get_alias_set(name) ⇒ Hash?

Returns { name:, groups:, created_at: } or nil if not found.

Returns:

  • (Hash, nil) —

    { name:, groups:, created_at: } or nil if not found



380
381
382
383
384
385
386
 
# File 'lib/wp2txt/metadata_index.rb', line 380

def get_alias_set(name)
  ensure_alias_table
  row = open_db.execute("SELECT name, groups, created_at FROM alias_sets WHERE name = ?", [name]).first
  return nil unless row

  { name: row[0], groups: JSON.parse(row[1]), created_at: row[2] }
end

#insert_batch(rows) ⇒ Object

Insert one scanned batch: categories:, sections:, hierarchy:



162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/wp2txt/metadata_index.rb', line 162

def insert_batch(rows)
  db = open_db
  db.transaction do
    stmt = db.prepare("INSERT OR IGNORE INTO pages (page_id, title, namespace, redirect_to, text_length) VALUES (?, ?, ?, ?, ?)")
    rows[:pages].each { |r| stmt.execute(r) }
    stmt.close

    stmt = db.prepare("INSERT INTO page_categories (page_id, category) VALUES (?, ?)")
    rows[:categories].each { |r| stmt.execute(r) }
    stmt.close

    stmt = db.prepare("INSERT INTO page_sections (page_id, heading, level, ord) VALUES (?, ?, ?, ?)")
    rows[:sections].each { |r| stmt.execute(r) }
    stmt.close

    stmt = db.prepare("INSERT INTO category_hierarchy (child, parent) VALUES (?, ?)")
    rows[:hierarchy].each { |r| stmt.execute(r) }
    stmt.close
  end
end

#list_alias_sets ⇒ Object 



388
389
390
391
392
393
 
# File 'lib/wp2txt/metadata_index.rb', line 388

def list_alias_sets
  ensure_alias_table
  open_db.execute("SELECT name, groups, created_at FROM alias_sets ORDER BY name").map do |name, groups, created_at|
    { name: name, group_count: JSON.parse(groups).size, created_at: created_at }
  end
end

#prepare_build! ⇒ Object

Build into a sidecar file and atomically rename in finalize_build!, so a failed multi-hour rebuild never destroys a working index



125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/wp2txt/metadata_index.rb', line 125

def prepare_build!
  FileUtils.mkdir_p(File.dirname(@db_path))
  close
  @build_path = "#{@db_path}.building"
  FileUtils.rm_f([@build_path, "#{@build_path}-wal", "#{@build_path}-shm"])
  db = open_db
  db.execute(<<~SQL)
    CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)
  SQL
  db.execute(<<~SQL)
    CREATE TABLE pages (
      page_id INTEGER PRIMARY KEY,
      title TEXT,
      namespace INTEGER,
      redirect_to TEXT,
      text_length INTEGER
    )
  SQL
  db.execute("CREATE TABLE page_categories (page_id INTEGER, category TEXT)")
  # ord: position of the section within the article, where 0 is the lead
  # text before the first heading. Lead rows are NOT stored here (they have
  # no heading), so ord starts at 1 — consistent with fts_map.ord in the
  # full-text DB, where the lead IS stored as ord 0.
  db.execute(<<~SQL)
    CREATE TABLE page_sections (
      page_id INTEGER,
      heading TEXT,
      level INTEGER,
      -- ord: section position in the article; 0 = lead text (not stored in
      -- this table), so headings start at 1. Same semantics as fts_map.ord.
      ord INTEGER
    )
  SQL
  db.execute("CREATE TABLE category_hierarchy (child TEXT, parent TEXT)")
end

#save_alias_set(name, groups) ⇒ Object

Save a named alias set. groups is an array of heading groups, e.g. [["あらすじ", "ストーリー", "物語"], ["脚注", "出典"]]

Raises:

  • (ArgumentError) 


368
369
370
371
372
373
374
375
376
377
 
# File 'lib/wp2txt/metadata_index.rb', line 368

def save_alias_set(name, groups)
  raise ArgumentError, "groups must be a non-empty array of arrays" unless groups.is_a?(Array) && !groups.empty? && groups.all? { |g| g.is_a?(Array) && !g.empty? }

  ensure_alias_table
  open_db.execute(
    "INSERT OR REPLACE INTO alias_sets (name, groups, created_at) VALUES (?, ?, ?)",
    [name, JSON.generate(groups), Time.now.utc.iso8601]
  )
  { name: name, groups: groups }
end

#section_cooccurrence(headings, category: nil, depth: 0) ⇒ Hash

Article counts, average positions, and pairwise co-occurrence for a set of headings. Synonymous headings almost never co-occur in the same article, so a high co-occurrence ratio is evidence AGAINST treating them as aliases.

Parameters:

  • headings (Array<String>) —

    headings to compare

  • category (String, nil) (defaults to: nil) —

    optional category scope

  • depth (Integer) (defaults to: 0) —

    category recursion depth

Returns:

  • (Hash) —

    { headings: [articles:, avg_position:], pairs: [b:, both:, cooccurrence_ratio:] }



317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
 
# File 'lib/wp2txt/metadata_index.rb', line 317

def section_cooccurrence(headings, category: nil, depth: 0)
  scope_cte = nil
  scope_cond = "p.namespace = #{NS_ARTICLE} AND p.redirect_to IS NULL"
  scope_params = []
  if category
    scope_cte, cond, scope_params = category_condition(category, depth)
    scope_cond += " AND #{cond}"
  end

  db = open_db
  with = scope_cte ? "WITH RECURSIVE #{scope_cte} " : ""
  # Placeholders bind positionally: with a recursive CTE the category `?`
  # sits inside the CTE (before any heading `?`); without one it sits
  # inside scope_cond (after the heading `?`)
  cte_params = scope_cte ? scope_params : []
  cond_params = scope_cte ? [] : scope_params

  heading_stats = headings.map do |h|
    row = db.execute(
      "#{with}SELECT COUNT(DISTINCT ps.page_id), AVG(ps.ord) FROM page_sections ps " \
      "JOIN pages p ON p.page_id = ps.page_id " \
      "WHERE ps.heading COLLATE NOCASE = ? AND #{scope_cond}",
      cte_params + [h] + cond_params
    ).first
    { heading: h, articles: row[0].to_i, avg_position: row[1]&.round(2) }
  end

  counts = heading_stats.to_h { |s| [s[:heading], s[:articles]] }
  pairs = headings.combination(2).map do |a, b|
    both = db.get_first_value(
      "#{with}SELECT COUNT(*) FROM (" \
      "SELECT ps.page_id FROM page_sections ps JOIN pages p ON p.page_id = ps.page_id " \
      "WHERE ps.heading COLLATE NOCASE = ? AND #{scope_cond} " \
      "INTERSECT " \
      "SELECT ps.page_id FROM page_sections ps JOIN pages p ON p.page_id = ps.page_id " \
      "WHERE ps.heading COLLATE NOCASE = ? AND #{scope_cond})",
      cte_params + [a] + cond_params + [b] + cond_params
    ).to_i
    min = [counts[a], counts[b]].min
    { a: a, b: b, both: both, cooccurrence_ratio: min.positive? ? (both.to_f / min).round(3) : 0.0 }
  end

  { headings: heading_stats, pairs: pairs }
end

#section_stats(category: nil, depth: 0, top_n: 50) ⇒ Object

Section heading frequencies across articles, optionally scoped to a category



289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
 
# File 'lib/wp2txt/metadata_index.rb', line 289

def section_stats(category: nil, depth: 0, top_n: 50)
  conds = ["p.namespace = #{NS_ARTICLE}", "p.redirect_to IS NULL"]
  params = []
  cte = nil
  if category
    cte, cond, cat_params = category_condition(category, depth)
    conds << cond
    params.concat(cat_params)
  end
  sql = +""
  sql << "WITH RECURSIVE #{cte} " if cte
  sql << <<~SQL
    SELECT ps.heading, COUNT(*) AS cnt
    FROM page_sections ps JOIN pages p ON p.page_id = ps.page_id
    WHERE #{conds.join(' AND ')}
    GROUP BY ps.heading ORDER BY cnt DESC, ps.heading LIMIT #{top_n.to_i}
  SQL
  open_db.execute(sql, params)
end

#stats ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/wp2txt/metadata_index.rb', line 97

def stats
  return nil unless File.exist?(@db_path)

  meta = read_metadata || {}
  {
    db_path: @db_path,
    db_size: File.size(@db_path),
    dump_name: meta[:dump_name],
    built_at: meta[:built_at],
    built_with: meta[:wp2txt_version],
    page_count: count_scalar("SELECT COUNT(*) FROM pages"),
    article_count: count_scalar("SELECT COUNT(*) FROM pages WHERE namespace = #{NS_ARTICLE} AND redirect_to IS NULL"),
    category_count: count_scalar("SELECT COUNT(DISTINCT category) FROM page_categories"),
    section_count: count_scalar("SELECT COUNT(*) FROM page_sections")
  }
end

#valid_for?(multistream_path) ⇒ Boolean

True if the index was built from the given (unchanged) multistream file

Returns:

  • (Boolean) 


88
89
90
91
92
93
94
95
 
# File 'lib/wp2txt/metadata_index.rb', line 88

def valid_for?(multistream_path)
  return false unless built?
  return false unless File.exist?(multistream_path)

  meta = read_metadata
  stat = File.stat(multistream_path)
  meta[:source_size].to_i == stat.size && meta[:source_mtime].to_i == stat.mtime.to_i
end