Class: Wp2txt::MetadataIndex

Inherits:
Object
  • Object
show all
Defined in:
lib/wp2txt/metadata_index.rb

Overview

Local metadata index (Tier 1) built from a multistream dump. Stores per-page categories, section headings, redirects, and the category hierarchy in SQLite, enabling offline exhaustive queries such as "all articles in category X that have a Plot section" without any API access.

Constant Summary collapse

SCHEMA_VERSION = 
2
CACHE_SUFFIX = 
"_meta.sqlite3"
NS_ARTICLE = 
0
NS_CATEGORY = 
14

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(db_path) ⇒ MetadataIndex

Returns a new instance of MetadataIndex.



27
28
29
30
 
# File 'lib/wp2txt/metadata_index.rb', line 27

def initialize(db_path)
  @db_path = db_path
  @db = nil
end

Instance Attribute Details

#db_path ⇒ Object (readonly)

Returns the value of attribute db_path.



25
26
27
 
# File 'lib/wp2txt/metadata_index.rb', line 25

def db_path
  @db_path
end

Class Method Details

.cached_candidates(lang, cache_dir: nil) ⇒ Array<Hash>

Built metadata DBs for one language found in a cache directory, most recently built first. Used for cross-dump ATTACH resolution (Corpus) and langlinks sanity checks (LanglinksImporter).

Returns:

  • (Array<Hash>) —

    [dump_name:, built_at:, built_with:]



58
59
60
61
62
63
64
65
66
67
 
# File 'lib/wp2txt/metadata_index.rb', line 58

def self.cached_candidates(lang, cache_dir: nil)
  dir = cache_dir || File.expand_path("~/.wp2txt/cache")
  Dir.glob(File.join(dir, "#{lang}wiki-*#{CACHE_SUFFIX}")).filter_map do |path|
    meta = read_metadata_file(path)
    next unless meta && meta[:schema_version].to_i == SCHEMA_VERSION && meta[:built_at]

    { db_path: path, dump_name: meta[:dump_name], built_at: meta[:built_at],
      built_with: meta[:wp2txt_version] }
  end.sort_by { |c| c[:built_at].to_s }.reverse
end

.clean_heading(text) ⇒ Object

Remove wiki markup from a heading ('''bold''', [[link|label]], HTML tags)



82
83
84
85
86
 
# File 'lib/wp2txt/metadata_index.rb', line 82

def self.clean_heading(text)
  t = text.gsub(/'{2,}/, "")
  t = t.gsub(/\[\[(?:[^\]|]*\|)?([^\]]*)\]\]/) { ::Regexp.last_match(1) }
  t.gsub(/<[^>]+>/, "").strip
end

.expand_section_names(name, alias_file: nil) ⇒ Object

Expand a section name to its full alias group (bidirectional): "Plot" => ["Plot", "Synopsis", ...]; "Synopsis" => same group



90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/wp2txt/metadata_index.rb', line 90

def self.expand_section_names(name, alias_file: nil)
  aliases = SectionExtractor::DEFAULT_ALIASES
  if alias_file
    custom = SectionExtractor.load_aliases_from_file(alias_file)
    aliases = aliases.merge(custom) unless custom.empty?
  end

  down = name.downcase
  aliases.each do |canonical, list|
    group = [canonical, *list]
    return group if group.any? { |g| g.downcase == down }
  end
  [name]
end

.normalize_category(name) ⇒ Object

Normalize a category name (same MediaWiki title rules as normalize_title)



50
51
52
 
# File 'lib/wp2txt/metadata_index.rb', line 50

def self.normalize_category(name)
  normalize_title(name)
end

.normalize_title(name) ⇒ Object

Normalize a page title the way MediaWiki treats titles: underscores to spaces, trimmed, first letter capitalized



42
43
44
45
46
47
 
# File 'lib/wp2txt/metadata_index.rb', line 42

def self.normalize_title(name)
  n = name.to_s.tr("_", " ").strip.squeeze(" ")
  return n if n.empty?

  n[0].upcase + n[1..].to_s
end

.path_for(multistream_path, cache_dir: nil) ⇒ Object

Default index location for a given multistream file (mirrors IndexCache naming)



33
34
35
36
37
38
 
# File 'lib/wp2txt/metadata_index.rb', line 33

def self.path_for(multistream_path, cache_dir: nil)
  dir = cache_dir || File.expand_path("~/.wp2txt/cache")
  basename = File.basename(multistream_path, ".*").sub(/\.xml\z/, "")
  path_hash = Digest::MD5.hexdigest(multistream_path)[0, 8]
  File.join(dir, "#{basename}_#{path_hash}#{CACHE_SUFFIX}")
end

.read_metadata_file(path) ⇒ Object

Light, read-only metadata table read for a DB file we do not manage



70
71
72
73
74
75
76
77
78
79
 
# File 'lib/wp2txt/metadata_index.rb', line 70

def self.read_metadata_file(path)
  db = SQLite3::Database.new(path, readonly: true)
  result = {}
  db.execute("SELECT key, value FROM metadata") { |key, value| result[key.to_sym] = value }
  result
rescue SQLite3::Exception
  nil
ensure
  db&.close
end

Instance Method Details

#built? ⇒ Boolean

True if the index file exists and has a compatible schema

Returns:

  • (Boolean) 


110
111
112
113
114
115
116
117
 
# File 'lib/wp2txt/metadata_index.rb', line 110

def built?
  return false unless File.exist?(@db_path)

  meta = read_metadata
  !meta.nil? && meta[:schema_version].to_i == SCHEMA_VERSION && !meta[:built_at].nil?
rescue SQLite3::Exception
  false
end

#categories_of(title) ⇒ Array<String>?

Categories of one article (by exact title)

Returns:

  • (Array<String>, nil) —

    category names, nil if the title is unknown



313
314
315
316
317
318
319
320
 
# File 'lib/wp2txt/metadata_index.rb', line 313

def categories_of(title)
  row = open_db.execute("SELECT page_id FROM pages WHERE title = ?", [title]).first
  return nil unless row

  open_db.execute(
    "SELECT category FROM page_categories WHERE page_id = ? ORDER BY category", [row[0]]
  ).map(&:first)
end

#category_tree(category, depth: 2) ⇒ Object

Subcategory tree starting at category, as [depth:, ...] (BFS order)



339
340
341
342
343
344
345
346
347
348
349
350
351
352
 
# File 'lib/wp2txt/metadata_index.rb', line 339

def category_tree(category, depth: 2)
  cat = self.class.normalize_category(category)
  sql = <<~SQL
    WITH RECURSIVE cat_tree(name, d) AS (
      SELECT ?, 0
      UNION
      SELECT ch.child, ct.d + 1
      FROM category_hierarchy ch JOIN cat_tree ct ON ch.parent = ct.name
      WHERE ct.d < #{depth.to_i}
    )
    SELECT name, MIN(d) FROM cat_tree GROUP BY name ORDER BY MIN(d), name
  SQL
  open_db.execute(sql, [cat]).map { |name, d| { name: name, depth: d } }
end

#close ⇒ Object 



164
165
166
167
 
# File 'lib/wp2txt/metadata_index.rb', line 164

def close
  @db&.close
  @db = nil
end

#count_articles(category: nil, depth: 0, categories: nil, category_match: nil, has_section: nil, sections: nil, alias_set: nil, use_aliases: true, alias_file: nil, title_match: nil) ⇒ Object

Count articles matching the same filters as find_articles



296
297
298
299
300
301
302
303
304
305
306
307
308
309
 
# File 'lib/wp2txt/metadata_index.rb', line 296

def count_articles(category: nil, depth: 0, categories: nil, category_match: nil,
                   has_section: nil, sections: nil, alias_set: nil,
                   use_aliases: true, alias_file: nil, title_match: nil)
  cte, where, params = build_article_query(
    category: category, depth: depth, categories: categories, category_match: category_match,
    has_section: has_section, sections: sections,
    alias_set: alias_set, use_aliases: use_aliases, alias_file: alias_file, title_match: title_match
  )
  sql = +""
  sql << "WITH RECURSIVE #{cte} " if cte
  sql << "SELECT COUNT(*) FROM pages p WHERE #{where}"

  open_db.get_first_value(sql, params).to_i
end

#delete_alias_set(name) ⇒ Object 



461
462
463
464
465
 
# File 'lib/wp2txt/metadata_index.rb', line 461

def delete_alias_set(name)
  ensure_alias_table
  open_db.execute("DELETE FROM alias_sets WHERE name = ?", [name])
  nil
end

#finalize_build!(source_path) ⇒ Object 



233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
 
# File 'lib/wp2txt/metadata_index.rb', line 233

def finalize_build!(source_path)
  db = open_db
  db.execute("CREATE INDEX IF NOT EXISTS idx_pages_title ON pages(title)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_pc_category ON page_categories(category)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_pc_page ON page_categories(page_id)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_ps_heading ON page_sections(heading COLLATE NOCASE)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_ps_page ON page_sections(page_id)")
  db.execute("CREATE INDEX IF NOT EXISTS idx_ch_parent ON category_hierarchy(parent)")

  stat = File.stat(source_path)
  dump_name = File.basename(source_path)[/\A[a-z0-9_\-]+?-\d{8}/] || File.basename(source_path)
  save_metadata(
    schema_version: SCHEMA_VERSION,
    wp2txt_version: Wp2txt::VERSION,
    source_path: source_path,
    source_size: stat.size,
    source_mtime: stat.mtime.to_i,
    dump_name: dump_name,
    built_at: Time.now.utc.iso8601
  )
  db.execute("ANALYZE")
  close
  if @build_path
    File.rename(@build_path, @db_path)
    FileUtils.rm_f(["#{@db_path}-wal", "#{@db_path}-shm"])
    @build_path = nil
  end
end

#find_articles(category: nil, depth: 0, categories: nil, category_match: nil, has_section: nil, sections: nil, alias_set: nil, use_aliases: true, alias_file: nil, title_match: nil, limit: 0, offset: 0) ⇒ Array<String>

Find article titles matching the given filters.

Parameters:

  • category (String, nil) (defaults to: nil) —

    category name (without namespace prefix)

  • depth (Integer) (defaults to: 0) —

    subcategory recursion depth (0 = exact category only)

  • has_section (String, nil) (defaults to: nil) —

    single section heading (alias-aware by default)

  • sections (Array<String>, nil) (defaults to: nil) —

    multiple headings (OR match, used as-is)

  • alias_set (String, nil) (defaults to: nil) —

    saved alias set name used to expand headings

  • use_aliases (Boolean) (defaults to: true) —

    expand has_section via built-in alias groups

  • alias_file (String, nil) (defaults to: nil) —

    custom alias YAML (merged with defaults)

  • title_match (String, nil) (defaults to: nil) —

    substring match on title

  • limit (Integer) (defaults to: 0) —

    max titles to return (0 = no limit)

  • offset (Integer) (defaults to: 0) —

    result offset

Returns:

  • (Array<String>) —

    matching titles ordered by page_id



278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
 
# File 'lib/wp2txt/metadata_index.rb', line 278

def find_articles(category: nil, depth: 0, categories: nil, category_match: nil,
                  has_section: nil, sections: nil, alias_set: nil,
                  use_aliases: true, alias_file: nil, title_match: nil, limit: 0, offset: 0)
  cte, where, params = build_article_query(
    category: category, depth: depth, categories: categories, category_match: category_match,
    has_section: has_section, sections: sections,
    alias_set: alias_set, use_aliases: use_aliases, alias_file: alias_file, title_match: title_match
  )
  sql = +""
  sql << "WITH RECURSIVE #{cte} " if cte
  sql << "SELECT p.title FROM pages p WHERE #{where} ORDER BY p.page_id"
  sql << " LIMIT #{limit.to_i}" if limit.to_i.positive?
  sql << " OFFSET #{offset.to_i}" if offset.to_i.positive?

  open_db.execute(sql, params).map { |row| row[0] }
end

#get_alias_set(name) ⇒ Hash?

Returns { name:, groups:, created_at: } or nil if not found.

Returns:

  • (Hash, nil) —

    { name:, groups:, created_at: } or nil if not found



446
447
448
449
450
451
452
 
# File 'lib/wp2txt/metadata_index.rb', line 446

def get_alias_set(name)
  ensure_alias_table
  row = open_db.execute("SELECT name, groups, created_at FROM alias_sets WHERE name = ?", [name]).first
  return nil unless row

  { name: row[0], groups: JSON.parse(row[1]), created_at: row[2] }
end

#insert_batch(rows) ⇒ Object

Insert one scanned batch: categories:, sections:, hierarchy:



212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
 
# File 'lib/wp2txt/metadata_index.rb', line 212

def insert_batch(rows)
  db = open_db
  db.transaction do
    stmt = db.prepare("INSERT OR IGNORE INTO pages (page_id, title, namespace, redirect_to, text_length) VALUES (?, ?, ?, ?, ?)")
    rows[:pages].each { |r| stmt.execute(r) }
    stmt.close

    stmt = db.prepare("INSERT INTO page_categories (page_id, category) VALUES (?, ?)")
    rows[:categories].each { |r| stmt.execute(r) }
    stmt.close

    stmt = db.prepare("INSERT INTO page_sections (page_id, heading, level, ord) VALUES (?, ?, ?, ?)")
    rows[:sections].each { |r| stmt.execute(r) }
    stmt.close

    stmt = db.prepare("INSERT INTO category_hierarchy (child, parent) VALUES (?, ?)")
    rows[:hierarchy].each { |r| stmt.execute(r) }
    stmt.close
  end
end

#langlinks_provenance ⇒ Object

Provenance of an imported langlinks table (nil when not imported). The langlinks table is an optional post-build addition (LanglinksImporter), so its absence does not affect built? or schema_version.



149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/wp2txt/metadata_index.rb', line 149

def langlinks_provenance
  return nil unless File.exist?(@db_path)

  meta = read_metadata
  return nil unless meta && meta[:langlinks_imported_at]

  { source: meta[:langlinks_source],
    source_size: meta[:langlinks_source_size].to_i,
    imported_at: meta[:langlinks_imported_at],
    imported_with: meta[:langlinks_wp2txt_version],
    lang_filter: meta[:langlinks_lang_filter],
    row_count: meta[:langlinks_row_count].to_i,
    skipped_invalid: meta[:langlinks_skipped_invalid].to_i }
end

#list_alias_sets ⇒ Object 



454
455
456
457
458
459
 
# File 'lib/wp2txt/metadata_index.rb', line 454

def list_alias_sets
  ensure_alias_table
  open_db.execute("SELECT name, groups, created_at FROM alias_sets ORDER BY name").map do |name, groups, created_at|
    { name: name, group_count: JSON.parse(groups).size, created_at: created_at }
  end
end

#prepare_build! ⇒ Object

Build into a sidecar file and atomically rename in finalize_build!, so a failed multi-hour rebuild never destroys a working index



175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
 
# File 'lib/wp2txt/metadata_index.rb', line 175

def prepare_build!
  FileUtils.mkdir_p(File.dirname(@db_path))
  close
  @build_path = "#{@db_path}.building"
  FileUtils.rm_f([@build_path, "#{@build_path}-wal", "#{@build_path}-shm"])
  db = open_db
  db.execute(<<~SQL)
    CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)
  SQL
  db.execute(<<~SQL)
    CREATE TABLE pages (
      page_id INTEGER PRIMARY KEY,
      title TEXT,
      namespace INTEGER,
      redirect_to TEXT,
      text_length INTEGER
    )
  SQL
  db.execute("CREATE TABLE page_categories (page_id INTEGER, category TEXT)")
  # ord: position of the section within the article, where 0 is the lead
  # text before the first heading. Lead rows are NOT stored here (they have
  # no heading), so ord starts at 1 — consistent with fts_map.ord in the
  # full-text DB, where the lead IS stored as ord 0.
  db.execute(<<~SQL)
    CREATE TABLE page_sections (
      page_id INTEGER,
      heading TEXT,
      level INTEGER,
      -- ord: section position in the article; 0 = lead text (not stored in
      -- this table), so headings start at 1. Same semantics as fts_map.ord.
      ord INTEGER
    )
  SQL
  db.execute("CREATE TABLE category_hierarchy (child TEXT, parent TEXT)")
end

#redirect_map(titles) ⇒ Hash

Look up titles in pages (existence + redirect target), batched to keep the IN clause small. Used by Corpus#extract_corpus titles: resolution.

Returns:

  • (Hash) —

    { title => redirect_to_or_nil } for the titles that exist



325
326
327
328
329
330
331
332
333
334
335
336
 
# File 'lib/wp2txt/metadata_index.rb', line 325

def redirect_map(titles)
  result = {}
  titles.each_slice(500) do |slice|
    placeholders = slice.map { "?" }.join(",")
    open_db.execute(
      "SELECT title, redirect_to FROM pages WHERE title IN (#{placeholders})", slice
    ).each do |title, redirect_to|
      result[title] = redirect_to
    end
  end
  result
end

#save_alias_set(name, groups) ⇒ Object

Save a named alias set. groups is an array of heading groups, e.g. [["あらすじ", "ストーリー", "物語"], ["脚注", "出典"]]

Raises:

  • (ArgumentError) 


434
435
436
437
438
439
440
441
442
443
 
# File 'lib/wp2txt/metadata_index.rb', line 434

def save_alias_set(name, groups)
  raise ArgumentError, "groups must be a non-empty array of arrays" unless groups.is_a?(Array) && !groups.empty? && groups.all? { |g| g.is_a?(Array) && !g.empty? }

  ensure_alias_table
  open_db.execute(
    "INSERT OR REPLACE INTO alias_sets (name, groups, created_at) VALUES (?, ?, ?)",
    [name, JSON.generate(groups), Time.now.utc.iso8601]
  )
  { name: name, groups: groups }
end

#section_cooccurrence(headings, category: nil, depth: 0) ⇒ Hash

Article counts, average positions, and pairwise co-occurrence for a set of headings. Synonymous headings almost never co-occur in the same article, so a high co-occurrence ratio is evidence AGAINST treating them as aliases.

Parameters:

  • headings (Array<String>) —

    headings to compare

  • category (String, nil) (defaults to: nil) —

    optional category scope

  • depth (Integer) (defaults to: 0) —

    category recursion depth

Returns:

  • (Hash) —

    { headings: [articles:, avg_position:], pairs: [b:, both:, cooccurrence_ratio:] }



383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
 
# File 'lib/wp2txt/metadata_index.rb', line 383

def section_cooccurrence(headings, category: nil, depth: 0)
  scope_cte = nil
  scope_cond = "p.namespace = #{NS_ARTICLE} AND p.redirect_to IS NULL"
  scope_params = []
  if category
    scope_cte, cond, scope_params = category_condition(category, depth)
    scope_cond += " AND #{cond}"
  end

  db = open_db
  with = scope_cte ? "WITH RECURSIVE #{scope_cte} " : ""
  # Placeholders bind positionally: with a recursive CTE the category `?`
  # sits inside the CTE (before any heading `?`); without one it sits
  # inside scope_cond (after the heading `?`)
  cte_params = scope_cte ? scope_params : []
  cond_params = scope_cte ? [] : scope_params

  heading_stats = headings.map do |h|
    row = db.execute(
      "#{with}SELECT COUNT(DISTINCT ps.page_id), AVG(ps.ord) FROM page_sections ps " \
      "JOIN pages p ON p.page_id = ps.page_id " \
      "WHERE ps.heading COLLATE NOCASE = ? AND #{scope_cond}",
      cte_params + [h] + cond_params
    ).first
    { heading: h, articles: row[0].to_i, avg_position: row[1]&.round(2) }
  end

  counts = heading_stats.to_h { |s| [s[:heading], s[:articles]] }
  pairs = headings.combination(2).map do |a, b|
    both = db.get_first_value(
      "#{with}SELECT COUNT(*) FROM (" \
      "SELECT ps.page_id FROM page_sections ps JOIN pages p ON p.page_id = ps.page_id " \
      "WHERE ps.heading COLLATE NOCASE = ? AND #{scope_cond} " \
      "INTERSECT " \
      "SELECT ps.page_id FROM page_sections ps JOIN pages p ON p.page_id = ps.page_id " \
      "WHERE ps.heading COLLATE NOCASE = ? AND #{scope_cond})",
      cte_params + [a] + cond_params + [b] + cond_params
    ).to_i
    min = [counts[a], counts[b]].min
    { a: a, b: b, both: both, cooccurrence_ratio: min.positive? ? (both.to_f / min).round(3) : 0.0 }
  end

  { headings: heading_stats, pairs: pairs }
end

#section_stats(category: nil, depth: 0, top_n: 50) ⇒ Object

Section heading frequencies across articles, optionally scoped to a category



355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
 
# File 'lib/wp2txt/metadata_index.rb', line 355

def section_stats(category: nil, depth: 0, top_n: 50)
  conds = ["p.namespace = #{NS_ARTICLE}", "p.redirect_to IS NULL"]
  params = []
  cte = nil
  if category
    cte, cond, cat_params = category_condition(category, depth)
    conds << cond
    params.concat(cat_params)
  end
  sql = +""
  sql << "WITH RECURSIVE #{cte} " if cte
  sql << <<~SQL
    SELECT ps.heading, COUNT(*) AS cnt
    FROM page_sections ps JOIN pages p ON p.page_id = ps.page_id
    WHERE #{conds.join(' AND ')}
    GROUP BY ps.heading ORDER BY cnt DESC, ps.heading LIMIT #{top_n.to_i}
  SQL
  open_db.execute(sql, params)
end

#stats ⇒ Object 



129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
 
# File 'lib/wp2txt/metadata_index.rb', line 129

def stats
  return nil unless File.exist?(@db_path)

  meta = read_metadata || {}
  {
    db_path: @db_path,
    db_size: File.size(@db_path),
    dump_name: meta[:dump_name],
    built_at: meta[:built_at],
    built_with: meta[:wp2txt_version],
    page_count: count_scalar("SELECT COUNT(*) FROM pages"),
    article_count: count_scalar("SELECT COUNT(*) FROM pages WHERE namespace = #{NS_ARTICLE} AND redirect_to IS NULL"),
    category_count: count_scalar("SELECT COUNT(DISTINCT category) FROM page_categories"),
    section_count: count_scalar("SELECT COUNT(*) FROM page_sections")
  }
end

#valid_for?(multistream_path) ⇒ Boolean

True if the index was built from the given (unchanged) multistream file

Returns:

  • (Boolean) 


120
121
122
123
124
125
126
127
 
# File 'lib/wp2txt/metadata_index.rb', line 120

def valid_for?(multistream_path)
  return false unless built?
  return false unless File.exist?(multistream_path)

  meta = read_metadata
  stat = File.stat(multistream_path)
  meta[:source_size].to_i == stat.size && meta[:source_mtime].to_i == stat.mtime.to_i
end