Class: Wp2txt::MultistreamIndex

Inherits:
Object
  • Object
show all
Defined in:
lib/wp2txt/multistream.rb

Overview

Manages multistream index for random access to Wikipedia dumps Supports SQLite caching for fast repeated access

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(index_path, use_cache: true, cache_dir: nil, target_titles: nil, show_progress: true) ⇒ MultistreamIndex

Initialize index with optional SQLite caching and early termination

Parameters:

  • index_path (String) —

    Path to the bz2 index file

  • use_cache (Boolean) (defaults to: true) —

    Whether to use SQLite cache (default: true)

  • cache_dir (String, nil) (defaults to: nil) —

    Directory for SQLite cache (default: ~/.wp2txt/cache)

  • target_titles (Array<String>, nil) (defaults to: nil) —

    If provided, stop parsing when all titles found

  • show_progress (Boolean) (defaults to: true) —

    Whether to show progress during parsing (default: true)



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/wp2txt/multistream.rb', line 60

def initialize(index_path, use_cache: true, cache_dir: nil, target_titles: nil, show_progress: true)
  @index_path = index_path
  @entries_by_title = {}
  @entries_by_id = {}
  @stream_offsets = []
  @show_progress = show_progress
  @target_titles = target_titles ? Set.new(target_titles) : nil
  @found_targets = Set.new if @target_titles

  # Try to load from cache first
  if use_cache && @target_titles.nil?
    @cache = IndexCache.new(index_path, cache_dir: cache_dir)
    if load_from_cache
      return
    end
  else
    @cache = nil
  end

  # Parse index file
  load_index

  # Save to cache for future use (only if full parse completed)
  if @cache && @target_titles.nil?
    save_to_cache
  end
end

Instance Attribute Details

#entries_by_id ⇒ Object (readonly)

Returns the value of attribute entries_by_id.



52
53
54
 
# File 'lib/wp2txt/multistream.rb', line 52

def entries_by_id
  @entries_by_id
end

#entries_by_title ⇒ Object (readonly)

Returns the value of attribute entries_by_title.



52
53
54
 
# File 'lib/wp2txt/multistream.rb', line 52

def entries_by_title
  @entries_by_title
end

#index_path ⇒ Object (readonly)

Returns the value of attribute index_path.



52
53
54
 
# File 'lib/wp2txt/multistream.rb', line 52

def index_path
  @index_path
end

#stream_offsets ⇒ Object (readonly)

Returns the value of attribute stream_offsets.



52
53
54
 
# File 'lib/wp2txt/multistream.rb', line 52

def stream_offsets
  @stream_offsets
end

Instance Method Details

#articles_in_stream(byte_offset) ⇒ Object

Get all articles in a specific stream (by byte offset)



107
108
109
 
# File 'lib/wp2txt/multistream.rb', line 107

def articles_in_stream(byte_offset)
  @entries_by_title.values.select { |e| e[:offset] == byte_offset }
end

#early_terminated? ⇒ Boolean

Check if early termination was triggered

Returns:

  • (Boolean) 


94
95
96
 
# File 'lib/wp2txt/multistream.rb', line 94

def early_terminated?
  @early_terminated == true
end

#find_by_id(page_id) ⇒ Object 



102
103
104
 
# File 'lib/wp2txt/multistream.rb', line 102

def find_by_id(page_id)
  @entries_by_id[page_id]
end

#find_by_title(title) ⇒ Object 



98
99
100
 
# File 'lib/wp2txt/multistream.rb', line 98

def find_by_title(title)
  @entries_by_title[title]
end

#first_articles(count) ⇒ Object

Get first N articles



123
124
125
 
# File 'lib/wp2txt/multistream.rb', line 123

def first_articles(count)
  @entries_by_title.keys.first(count)
end

#loaded_from_cache? ⇒ Boolean

Check if this index was loaded from cache

Returns:

  • (Boolean) 


89
90
91
 
# File 'lib/wp2txt/multistream.rb', line 89

def loaded_from_cache?
  @loaded_from_cache == true
end

#random_articles(count) ⇒ Object

Get N random articles



118
119
120
 
# File 'lib/wp2txt/multistream.rb', line 118

def random_articles(count)
  @entries_by_title.keys.sample(count)
end

#size ⇒ Object

Total number of articles



128
129
130
 
# File 'lib/wp2txt/multistream.rb', line 128

def size
  @entries_by_title.size
end

#stream_offset_for(title) ⇒ Object

Get stream offset for a given article



112
113
114
115
 
# File 'lib/wp2txt/multistream.rb', line 112

def stream_offset_for(title)
  entry = find_by_title(title)
  entry ? entry[:offset] : nil
end