Module: Wp2txt::CLI

Defined in:
lib/wp2txt/cli.rb

Overview

CLI option parsing and validation

Class Method Summary collapse

Class Method Details

.config ⇒ Config

Get current config (lazy load)

Returns:

  • (Config) —

    Configuration object



29
30
31
 
# File 'lib/wp2txt/cli.rb', line 29

def config
  @config ||= load_config
end

.default_cache_dir ⇒ Object

Get default cache directory



314
315
316
 
# File 'lib/wp2txt/cli.rb', line 314

def default_cache_dir
  Config::DEFAULT_CACHE_DIR
end

.load_config(config_path = nil) ⇒ Config

Load configuration

Returns:

  • (Config) —

    Configuration object



22
23
24
25
 
# File 'lib/wp2txt/cli.rb', line 22

def load_config(config_path = nil)
  path = config_path || Config.default_path
  @config = Config.load(path)
end

.max_processors ⇒ Integer

Get the optimal number of processors for this system Based on CPU cores and available memory

Returns:

  • (Integer) —

    Optimal number of parallel processes



16
17
18
 
# File 'lib/wp2txt/cli.rb', line 16

def max_processors
  MemoryMonitor.optimal_processes
end

.parse_article_list(articles_str) ⇒ Object

Parse article list from comma-separated string



300
301
302
303
 
# File 'lib/wp2txt/cli.rb', line 300

def parse_article_list(articles_str)
  return [] if articles_str.nil? || articles_str.empty?
  articles_str.split(",").map(&:strip).reject(&:empty?)
end

.parse_options(args) ⇒ Hash

Parse command line options

Parameters:

  • args (Array<String>) —

    Command line arguments

Returns:

  • (Hash) —

    Parsed options



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/wp2txt/cli.rb', line 36

def parse_options(args)
  # Pre-load config for defaults
  cfg = config

  opts = Optimist.options(args) do
    version Wp2txt::VERSION
    banner <<~BANNER
      WP2TXT extracts plain text data from Wikipedia dump files.

      Usage:
        wp2txt --lang=ja [options]                    # Auto-download and process
        wp2txt --input=FILE [options]                 # Process local file
        wp2txt --lang=en --from-category="Cities" -o ./output  # Extract category
        wp2txt --lang=en --from-category="Cities" --dry-run    # Preview only
        wp2txt --cache-status                         # Show cache status
        wp2txt --cache-clear [--lang=CODE]            # Clear cache
        wp2txt --config-init                          # Create default config file

      Options:
    BANNER

    # Input source (one of --input or --lang required, unless cache operations)
    opt :input, "Path to compressed file (bz2) or XML file",
        type: String, short: "-i"
    opt :lang, "Wikipedia language code (e.g., ja, en, de) for auto-download",
        type: String, short: "-L"
    opt :articles, "Specific article titles to extract (comma-separated, requires --lang)",
        type: String, short: "-A"
    opt :from_category, "Extract articles from Wikipedia category (requires --lang)",
        type: String, short: "-G"
    opt :depth, "Subcategory recursion depth for --from-category (0 = no recursion)",
        default: cfg.default_depth, type: Integer, short: "-D"
    opt :yes, "Skip confirmation prompt for category extraction",
        default: false, short: "-y"
    opt :dry_run, "Preview category extraction without downloading",
        default: false
    opt :update_cache, "Force refresh of cached dump files",
        default: false, short: "-U"

    # Output options
    opt :output_dir, "Path to output directory",
        default: Dir.pwd, type: String, short: "-o"
    opt :format, "Output format: text or json (JSONL)",
        default: cfg.default_format, short: "-j"

    # Cache management
    opt :cache_dir, "Cache directory for downloaded dumps",
        default: cfg.cache_directory, type: String
    opt :cache_status, "Show cache status and exit",
        default: false
    opt :cache_clear, "Clear cache and exit",
        default: false
    opt :config_init, "Create default configuration file (~/.wp2txt/config.yml)",
        default: false
    opt :config_path, "Path to configuration file",
        type: String

    # Processing options
    opt :category, "Show article category information",
        default: true, short: "-a"
    opt :category_only, "Extract only article title and categories",
        default: false, short: "-g"
    opt :summary_only, "Extract only title, categories, and summary",
        default: false, short: "-s"
    opt :metadata_only, "Extract only title, section headings, and categories (for analysis)",
        default: false, short: "-M"

    # Section extraction options
    opt :sections, "Extract specific sections (comma-separated, 'summary' for lead text)",
        type: String, short: "-S"
    opt :section_output, "Section output mode: 'structured' (default) or 'combined'",
        default: "structured"
    opt :min_section_length, "Minimum section length in characters (shorter sections become null)",
        default: 0, type: Integer
    opt :skip_empty, "Skip articles with no matching sections",
        default: false
    opt :alias_file, "Custom section alias definitions file (YAML format)",
        type: String
    opt :no_section_aliases, "Disable section alias matching (exact match only)",
        default: false
    opt :section_stats, "Collect and output section heading statistics (JSON)",
        default: false
    opt :show_matched_sections, "Include matched_sections field in JSON output (shows actual headings)",
        default: false

    opt :file_size, "Approximate size (in MB) of each output file (0 for single file)",
        default: 10, short: "-f"
    opt :num_procs, "Number of parallel processes (auto-detected based on CPU/memory)",
        type: Integer, short: "-n"
    opt :title, "Keep page titles in output",
        default: true, short: "-t"
    opt :heading, "Keep section titles in output",
        default: true, short: "-d"
    opt :list, "Keep unprocessed list items in output",
        default: false, short: "-l"
    opt :table, "Keep wiki table content in output",
        default: false
    opt :pre, "Keep preformatted text blocks in output",
        default: false, short: "-p"
    opt :ref, "Keep reference notations [ref]...[/ref]",
        default: false, short: "-r"
    opt :multiline, "Keep multi-line templates in output",
        default: false
    opt :redirect, "Show redirect destination",
        default: false, short: "-e"
    opt :marker, "Show symbols prefixed to list items",
        default: true, short: "-m"
    opt :markers, "Content type markers (math,code,chem,table,score,timeline,graph,ipa or 'all')",
        default: "all", short: "-k"
    opt :extract_citations, "Extract formatted citations instead of removing them",
        default: false, short: "-C"
    opt :expand_templates, "Expand common templates (birth date, convert, etc.) to readable text",
        default: true, short: "-E"
    opt :bz2_gem, "Use Ruby's bzip2-ruby gem instead of system command",
        default: false, short: "-b"
    opt :ractor, "Use Ractor for parallel processing (Ruby 4.0+, streaming mode only)",
        default: false, short: "-R"
    opt :no_turbo, "Disable turbo mode (use streaming instead, saves disk space)",
        default: false

    # Output control
    opt :quiet, "Suppress progress output (only show errors and final result)",
        default: false, short: "-q"
    opt :no_color, "Disable colored output (also respects NO_COLOR env variable)",
        default: false

    # Deprecated options
    opt :convert, "[DEPRECATED] No longer needed",
        default: true, short: "-c"
    opt :del_interfile, "[DEPRECATED] Intermediate files no longer created",
        default: false, short: "-x"
  end

  validate_options!(opts)
  opts
end

.valid_language_code?(code) ⇒ Boolean

Check if a language code is valid Valid codes: 2-10 lowercase letters (e.g., en, ja, simple, zh-yue)

Returns:

  • (Boolean) 


307
308
309
310
311
 
# File 'lib/wp2txt/cli.rb', line 307

def valid_language_code?(code)
  return false if code.nil? || code.empty?
  # Allow codes like: en, ja, zh, simple, zh-yue, etc.
  code.match?(/\A[a-z]{2,10}(-[a-z]{2,10})?\z/)
end

.validate_options!(opts) ⇒ Object

Validate parsed options



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/wp2txt/cli.rb', line 174

def validate_options!(opts)
  # Cache and config operations don't need input/lang
  return if opts[:cache_status] || opts[:cache_clear] || opts[:config_init]

  # Either --input or --lang is required
  if opts[:input].nil? && opts[:lang].nil?
    Optimist.die "Either --input or --lang is required"
  end

  # Cannot specify both --input and --lang
  if opts[:input] && opts[:lang]
    Optimist.die "Cannot specify both --input and --lang"
  end

  # --articles requires --lang
  if opts[:articles] && opts[:lang].nil?
    Optimist.die "--articles requires --lang"
  end

  # --articles cannot be used with --input
  if opts[:articles] && opts[:input]
    Optimist.die "--articles cannot be used with --input"
  end

  # --from-category requires --lang
  if opts[:from_category] && opts[:lang].nil?
    Optimist.die "--from-category requires --lang"
  end

  # --from-category cannot be used with --input
  if opts[:from_category] && opts[:input]
    Optimist.die "--from-category cannot be used with --input"
  end

  # --from-category cannot be used with --articles
  if opts[:from_category] && opts[:articles]
    Optimist.die "--from-category cannot be used with --articles"
  end

  # --depth must be >= 0
  if opts[:depth] < 0
    Optimist.die :depth, "must be 0 or greater"
  end

  # Warn if depth > 3 (can result in many articles)
  if opts[:depth] > 3
    warn "Warning: --depth > 3 may result in a very large number of articles"
  end

  # --dry-run only makes sense with --from-category
  if opts[:dry_run] && opts[:from_category].nil?
    Optimist.die "--dry-run requires --from-category"
  end

  # --yes only makes sense with --from-category
  if opts[:yes] && opts[:from_category].nil?
    Optimist.die "--yes requires --from-category"
  end

  # Validate --input exists
  if opts[:input] && !File.exist?(opts[:input])
    Optimist.die :input, "file does not exist"
  end

  # Validate language code
  if opts[:lang] && !valid_language_code?(opts[:lang])
    Optimist.die :lang, "invalid language code format"
  end

  # Validate output directory exists
  unless File.exist?(opts[:output_dir])
    Optimist.die :output_dir, "directory does not exist"
  end

  # Validate format
  unless %w[text json].include?(opts[:format].to_s.downcase)
    Optimist.die :format, "must be 'text' or 'json'"
  end

  # Validate file_size
  Optimist.die :file_size, "must be 0 or larger" if opts[:file_size] < 0

  # Validate --alias-file exists and is valid YAML
  if opts[:alias_file]
    unless File.exist?(opts[:alias_file])
      Optimist.die :alias_file, "file does not exist"
    end
    begin
      require "yaml"
      YAML.safe_load(File.read(opts[:alias_file]), permitted_classes: [Symbol])
    rescue Psych::SyntaxError => e
      Optimist.die :alias_file, "invalid YAML syntax: #{e.message}"
    end
  end

  # Extraction modes are mutually exclusive
  extraction_modes = []
  extraction_modes << "--category-only" if opts[:category_only]
  extraction_modes << "--summary-only" if opts[:summary_only]
  extraction_modes << "--metadata-only" if opts[:metadata_only]
  if extraction_modes.size > 1
    Optimist.die "#{extraction_modes.join(', ')} cannot be combined (choose one extraction mode)"
  end

  # --sections conflicts with extraction modes
  if opts[:sections] && extraction_modes.any?
    Optimist.die "--sections cannot be used with #{extraction_modes.first}"
  end

  # --section-stats is a standalone mode
  if opts[:section_stats]
    if opts[:sections]
      Optimist.die "--section-stats cannot be used with --sections"
    end
    if extraction_modes.any?
      Optimist.die "--section-stats cannot be used with #{extraction_modes.first}"
    end
  end

  # --show-matched-sections only works with JSON format
  if opts[:show_matched_sections] && opts[:format].to_s.downcase != "json"
    Optimist.die "--show-matched-sections requires --format json"
  end
end