Class: Wp2txt::GlobalDataCache

Inherits:
Object
  • Object
show all
Defined in:
lib/wp2txt/global_data_cache.rb

Overview

SQLite-based cache for global data files (templates, mediawiki aliases, entities) Dramatically speeds up startup by avoiding JSON parsing overhead

Constant Summary collapse

CACHE_VERSION = 
1
DEFAULT_CACHE_DIR = 
File.expand_path("~/.wp2txt/cache")
DATA_SOURCES =

Data categories and their source paths Note: html_entities_combined has no direct source (derived from html_entities + wikipedia_entities)

{
  mediawiki: "mediawiki_aliases.json",
  template: "template_aliases.json",
  html_entities: "html_entities.json",
  wikipedia_entities: "wikipedia_entities.json",
  language_metadata: "language_metadata.json",
  language_tiers: "language_tiers.json"
}.freeze
DERIVED_SOURCES =

Categories that are derived (combined from multiple sources) These are validated by checking their source files

{
  html_entities_combined: [:html_entities, :wikipedia_entities]
}.freeze

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.cache_dir ⇒ Object

Returns the value of attribute cache_dir.



33
34
35
 
# File 'lib/wp2txt/global_data_cache.rb', line 33

def cache_dir
  @cache_dir
end

.enabled ⇒ Object

Returns the value of attribute enabled.



33
34
35
 
# File 'lib/wp2txt/global_data_cache.rb', line 33

def enabled
  @enabled
end

Class Method Details

.cache_path ⇒ Object 



40
41
42
43
 
# File 'lib/wp2txt/global_data_cache.rb', line 40

def cache_path
  @cache_dir ||= DEFAULT_CACHE_DIR
  File.join(@cache_dir, "global_data.sqlite3")
end

.cache_valid? ⇒ Boolean

Check if cache is valid for all source files

Returns:

  • (Boolean) 


50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/wp2txt/global_data_cache.rb', line 50

def cache_valid?
  return false unless @enabled
  return false unless File.exist?(cache_path)

  begin
    db = open_db
    DATA_SOURCES.each do |category, filename|
      source_path = File.join(data_dir, filename)
      next unless File.exist?(source_path)

      meta = load_metadata(db, category)
      return false unless meta

      # Check version
      return false if meta[:cache_version].to_i != CACHE_VERSION

      # Check source file hasn't changed
      source_stat = File.stat(source_path)
      return false if meta[:source_mtime].to_i != source_stat.mtime.to_i
      return false if meta[:source_size].to_i != source_stat.size
    end
    true
  rescue SQLite3::Exception
    false
  ensure
    db&.close
  end
end

.category_valid?(category) ⇒ Boolean

Check if a specific category's cache is valid

Returns:

  • (Boolean) 


80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/wp2txt/global_data_cache.rb', line 80

def category_valid?(category)
  return false unless @enabled
  return false unless File.exist?(cache_path)

  # For derived categories, check source categories
  if DERIVED_SOURCES.key?(category)
    return DERIVED_SOURCES[category].all? { |src| category_valid?(src) }
  end

  # For unknown categories (not in DATA_SOURCES), just check if it exists in cache
  filename = DATA_SOURCES[category]
  unless filename
    begin
      db = open_db
      row = db.get_first_row("SELECT 1 FROM global_data WHERE category = ?", [category.to_s])
      return !row.nil?
    rescue SQLite3::Exception
      return false
    ensure
      db&.close
    end
  end

  # For known data sources, validate against source file
  begin
    db = open_db
    source_path = File.join(data_dir, filename)
    return true unless File.exist?(source_path)

    meta = load_metadata(db, category)
    return false unless meta
    return false if meta[:cache_version].to_i != CACHE_VERSION

    source_stat = File.stat(source_path)
    return false if meta[:source_mtime].to_i != source_stat.mtime.to_i
    return false if meta[:source_size].to_i != source_stat.size

    true
  rescue SQLite3::Exception
    false
  ensure
    db&.close
  end
end

.clear! ⇒ Object

Clear cache



272
273
274
 
# File 'lib/wp2txt/global_data_cache.rb', line 272

def clear!
  FileUtils.rm_f(cache_path)
end

.configure(cache_dir: nil, enabled: true) ⇒ Object 



35
36
37
38
 
# File 'lib/wp2txt/global_data_cache.rb', line 35

def configure(cache_dir: nil, enabled: true)
  @cache_dir = cache_dir || DEFAULT_CACHE_DIR
  @enabled = enabled
end

.data_dir ⇒ Object 



45
46
47
 
# File 'lib/wp2txt/global_data_cache.rb', line 45

def data_dir
  File.join(__dir__, "data")
end

.load(category) ⇒ Hash?

Load data from cache

Parameters:

  • category (Symbol) —

    Data category (:mediawiki, :template, etc.)

Returns:

  • (Hash, nil) —

    Parsed data or nil if not cached or invalid



128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/wp2txt/global_data_cache.rb', line 128

def load(category)
  return nil unless @enabled
  return nil unless File.exist?(cache_path)
  return nil unless category_valid?(category)

  begin
    db = open_db
    row = db.get_first_row(
      "SELECT data FROM global_data WHERE category = ?",
      [category.to_s]
    )
    return nil unless row

    JSON.parse(row[0])
  rescue SQLite3::Exception, JSON::ParserError
    nil
  ensure
    db&.close
  end
end

.load_all ⇒ Hash

Load all data categories at once (more efficient)

Returns:

  • (Hash) —

    { category => data }



208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
 
# File 'lib/wp2txt/global_data_cache.rb', line 208

def load_all
  return {} unless @enabled
  return {} unless File.exist?(cache_path)

  result = {}
  begin
    db = open_db
    db.execute("SELECT category, data FROM global_data") do |row|
      category = row[0].to_sym
      result[category] = JSON.parse(row[1])
    end
    result
  rescue SQLite3::Exception, JSON::ParserError
    {}
  ensure
    db&.close
  end
end

.save(category, data) ⇒ Object

Save data to cache

Parameters:

  • category (Symbol) —

    Data category

  • data (Hash) —

    Data to cache



152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
 
# File 'lib/wp2txt/global_data_cache.rb', line 152

def save(category, data)
  return unless @enabled

  FileUtils.mkdir_p(File.dirname(cache_path))

  begin
    db = open_db
    create_schema(db)

    db.execute(
      "INSERT OR REPLACE INTO global_data (category, data, updated_at) VALUES (?, ?, ?)",
      [category.to_s, JSON.generate(data), Time.now.to_i]
    )

    # For derived categories, save metadata from source files
    if DERIVED_SOURCES.key?(category)
      DERIVED_SOURCES[category].each do |src_category|
        filename = DATA_SOURCES[src_category]
        next unless filename

        source_path = File.join(data_dir, filename)
        next unless File.exist?(source_path)

        source_stat = File.stat(source_path)
        save_metadata(db, src_category,
          source_path: source_path,
          source_mtime: source_stat.mtime.to_i,
          source_size: source_stat.size,
          cache_version: CACHE_VERSION
        )
      end
    else
      # For regular categories, save metadata from the source file
      filename = DATA_SOURCES[category]
      if filename
        source_path = File.join(data_dir, filename)
        if File.exist?(source_path)
          source_stat = File.stat(source_path)
          save_metadata(db, category,
            source_path: source_path,
            source_mtime: source_stat.mtime.to_i,
            source_size: source_stat.size,
            cache_version: CACHE_VERSION
          )
        end
      end
    end
  rescue SQLite3::Exception => e
    warn "GlobalDataCache: Failed to save #{category}: #{e.message}"
  ensure
    db&.close
  end
end

.save_all(data_hash) ⇒ Object

Save all data categories at once

Parameters:

  • data_hash (Hash) —

    { category => data }



229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
 
# File 'lib/wp2txt/global_data_cache.rb', line 229

def save_all(data_hash)
  return unless @enabled

  FileUtils.mkdir_p(File.dirname(cache_path))

  begin
    db = open_db
    create_schema(db)

    db.execute("BEGIN TRANSACTION")

    data_hash.each do |category, data|
      db.execute(
        "INSERT OR REPLACE INTO global_data (category, data, updated_at) VALUES (?, ?, ?)",
        [category.to_s, JSON.generate(data), Time.now.to_i]
      )

      # Only save metadata if this is a known data source
      filename = DATA_SOURCES[category]
      if filename
        source_path = File.join(data_dir, filename)
        if File.exist?(source_path)
          source_stat = File.stat(source_path)
          save_metadata(db, category,
            source_path: source_path,
            source_mtime: source_stat.mtime.to_i,
            source_size: source_stat.size,
            cache_version: CACHE_VERSION
          )
        end
      end
    end

    db.execute("COMMIT")
  rescue SQLite3::Exception => e
    db&.execute("ROLLBACK") rescue nil
    warn "GlobalDataCache: Failed to save all: #{e.message}"
  ensure
    db&.close
  end
end

.stats ⇒ Object

Get cache statistics



277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
 
# File 'lib/wp2txt/global_data_cache.rb', line 277

def stats
  return nil unless File.exist?(cache_path)

  begin
    db = open_db
    categories = db.execute("SELECT category, LENGTH(data), updated_at FROM global_data")

    {
      cache_path: cache_path,
      cache_size: File.size(cache_path),
      categories: categories.map do |row|
        {
          category: row[0],
          data_size: row[1],
          updated_at: row[2] ? Time.at(row[2]) : nil
        }
      end
    }
  rescue SQLite3::Exception
    nil
  ensure
    db&.close
  end
end