Class: Wp2txt::CategoryCache

Inherits:
Object
  • Object
show all
Defined in:
lib/wp2txt/category_cache.rb

Overview

SQLite-based cache for Wikipedia category hierarchy and members Dramatically speeds up repeated category extraction operations

Constant Summary collapse

CACHE_VERSION = 
1
DEFAULT_CACHE_DIR = 
File.expand_path("~/.wp2txt/cache")

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(lang, cache_dir: nil, expiry_days: nil) ⇒ CategoryCache

Returns a new instance of CategoryCache.



17
18
19
20
21
22
23
24
 
# File 'lib/wp2txt/category_cache.rb', line 17

def initialize(lang, cache_dir: nil, expiry_days: nil)
  @lang = lang.to_s
  @cache_dir = cache_dir || DEFAULT_CACHE_DIR
  @expiry_days = expiry_days || DEFAULT_CATEGORY_CACHE_EXPIRY_DAYS
  @cache_path = File.join(@cache_dir, "categories_#{@lang}.sqlite3")
  @db = nil
  ensure_schema
end

Instance Attribute Details

#cache_path ⇒ Object (readonly)

Returns the value of attribute cache_path.



15
16
17
 
# File 'lib/wp2txt/category_cache.rb', line 15

def cache_path
  @cache_path
end

#expiry_days ⇒ Object (readonly)

Returns the value of attribute expiry_days.



15
16
17
 
# File 'lib/wp2txt/category_cache.rb', line 15

def expiry_days
  @expiry_days
end

#lang ⇒ Object (readonly)

Returns the value of attribute lang.



15
16
17
 
# File 'lib/wp2txt/category_cache.rb', line 15

def lang
  @lang
end

Instance Method Details

#cached?(category_name) ⇒ Boolean

Check if a category is cached and fresh

Parameters:

  • category_name (String) —

    Category name (without "Category:" prefix)

Returns:

  • (Boolean) 


29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/wp2txt/category_cache.rb', line 29

def cached?(category_name)
  open_db
  row = @db.get_first_row(
    "SELECT cached_at FROM categories WHERE name = ?",
    [normalize_name(category_name)]
  )
  return false unless row

  cached_at = row[0]
  return false unless cached_at

  # Check freshness
  Time.at(cached_at) > Time.now - (@expiry_days * SECONDS_PER_DAY)
rescue SQLite3::Exception
  false
end

#cleanup_expired! ⇒ Object

Clear expired entries



187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
 
# File 'lib/wp2txt/category_cache.rb', line 187

def cleanup_expired!
  open_db
  cutoff = Time.now.to_i - (@expiry_days * SECONDS_PER_DAY)

  @db.execute("BEGIN TRANSACTION")

  # Get expired categories
  expired = []
  @db.execute("SELECT name FROM categories WHERE cached_at < ?", [cutoff]) do |row|
    expired << row[0]
  end

  # Delete expired data
  expired.each do |name|
    @db.execute("DELETE FROM category_pages WHERE category_name = ?", [name])
    @db.execute("DELETE FROM category_hierarchy WHERE parent_name = ?", [name])
    @db.execute("DELETE FROM categories WHERE name = ?", [name])
  end

  @db.execute("COMMIT")

  expired.size
rescue SQLite3::Exception
  @db&.execute("ROLLBACK") rescue nil
  0
end

#clear! ⇒ Object

Clear all cached data



180
181
182
183
184
 
# File 'lib/wp2txt/category_cache.rb', line 180

def clear!
  close_db
  FileUtils.rm_f(@cache_path)
  ensure_schema
end

#close ⇒ Object

Close database connection



215
216
217
 
# File 'lib/wp2txt/category_cache.rb', line 215

def close
  close_db
end

#get(category_name) ⇒ Hash?

Get category data from cache

Parameters:

  • category_name (String) —

    Category name

Returns:

  • (Hash, nil) —

    { pages: [...], subcats: [...] } or nil if not cached



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/wp2txt/category_cache.rb', line 49

def get(category_name)
  return nil unless cached?(category_name)

  name = normalize_name(category_name)
  open_db

  pages = []
  subcats = []

  # Get pages
  @db.execute(
    "SELECT page_title FROM category_pages WHERE category_name = ?",
    [name]
  ) do |row|
    pages << row[0]
  end

  # Get subcategories
  @db.execute(
    "SELECT child_name FROM category_hierarchy WHERE parent_name = ?",
    [name]
  ) do |row|
    subcats << row[0]
  end

  { pages: pages, subcats: subcats }
rescue SQLite3::Exception
  nil
end

#get_all_pages(category_name, max_depth: 0, visited: nil) ⇒ Array<String>

Get all pages in a category tree (recursive)

Parameters:

  • category_name (String) —

    Root category name

  • max_depth (Integer) (defaults to: 0) —

    Maximum recursion depth (0 = no recursion)

  • visited (Set) (defaults to: nil) —

    Already visited categories (for cycle detection)

Returns:

  • (Array<String>) —

    All article titles



124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
 
# File 'lib/wp2txt/category_cache.rb', line 124

def get_all_pages(category_name, max_depth: 0, visited: nil)
  visited ||= Set.new
  name = normalize_name(category_name)
  return [] if visited.include?(name)

  visited << name
  data = get(name)
  return [] unless data

  pages = data[:pages].dup

  if max_depth > 0
    data[:subcats].each do |subcat|
      pages.concat(get_all_pages(subcat, max_depth: max_depth - 1, visited: visited))
    end
  end

  pages.uniq
end

#get_tree(category_name, max_depth: 0) ⇒ Hash

Get category tree structure

Parameters:

  • category_name (String) —

    Root category name

  • max_depth (Integer) (defaults to: 0) —

    Maximum recursion depth

Returns:

  • (Hash) —

    Tree structure with category info



148
149
150
 
# File 'lib/wp2txt/category_cache.rb', line 148

def get_tree(category_name, max_depth: 0)
  build_tree(category_name, max_depth, Set.new)
end

#save(category_name, pages, subcats) ⇒ Object

Save category data to cache

Parameters:

  • category_name (String) —

    Category name

  • pages (Array<String>) —

    Article titles in this category

  • subcats (Array<String>) —

    Subcategory names



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/wp2txt/category_cache.rb', line 83

def save(category_name, pages, subcats)
  name = normalize_name(category_name)
  open_db

  @db.execute("BEGIN TRANSACTION")

  # Update or insert category
  @db.execute(
    "INSERT OR REPLACE INTO categories (name, page_count, subcat_count, cached_at) VALUES (?, ?, ?, ?)",
    [name, pages.size, subcats.size, Time.now.to_i]
  )

  # Clear old pages and hierarchy
  @db.execute("DELETE FROM category_pages WHERE category_name = ?", [name])
  @db.execute("DELETE FROM category_hierarchy WHERE parent_name = ?", [name])

  # Insert pages
  unless pages.empty?
    stmt = @db.prepare("INSERT INTO category_pages (category_name, page_title) VALUES (?, ?)")
    pages.each { |page| stmt.execute([name, page]) }
    stmt.close
  end

  # Insert subcategories
  unless subcats.empty?
    stmt = @db.prepare("INSERT INTO category_hierarchy (parent_name, child_name) VALUES (?, ?)")
    subcats.each { |subcat| stmt.execute([name, normalize_name(subcat)]) }
    stmt.close
  end

  @db.execute("COMMIT")
rescue SQLite3::Exception => e
  @db&.execute("ROLLBACK") rescue nil
  warn "CategoryCache: Failed to save #{category_name}: #{e.message}"
end

#stats ⇒ Hash

Get statistics for all cached categories

Returns:

  • (Hash) —

    Statistics



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/wp2txt/category_cache.rb', line 154

def stats
  open_db

  total_categories = @db.get_first_value("SELECT COUNT(*) FROM categories")
  total_pages = @db.get_first_value("SELECT COUNT(*) FROM category_pages")
  total_relations = @db.get_first_value("SELECT COUNT(*) FROM category_hierarchy")

  oldest_cache = @db.get_first_value("SELECT MIN(cached_at) FROM categories")
  newest_cache = @db.get_first_value("SELECT MAX(cached_at) FROM categories")

  {
    lang: @lang,
    cache_path: @cache_path,
    cache_size: File.exist?(@cache_path) ? File.size(@cache_path) : 0,
    total_categories: total_categories || 0,
    total_pages: total_pages || 0,
    total_relations: total_relations || 0,
    oldest_cache: oldest_cache ? Time.at(oldest_cache) : nil,
    newest_cache: newest_cache ? Time.at(newest_cache) : nil,
    expiry_days: @expiry_days
  }
rescue SQLite3::Exception
  { lang: @lang, error: "Failed to read stats" }
end