Class: Archaeo::BulkDownloader

Inherits:
Object
  • Object
show all
Defined in:
lib/archaeo/bulk_downloader.rb

Overview

Downloads all archived snapshots of a URL with resume support.

Queries the CDX API for matching snapshots, fetches each page, and saves content to disk. Progress is tracked in a state file for interrupted download recovery.

Instance Method Summary collapse

Constructor Details

#initialize(client: HttpClient.new, output_dir: "archive", cdx_api: nil, concurrency: 1, on_error: nil) ⇒ BulkDownloader

Returns a new instance of BulkDownloader.



17
18
19
20
21
22
23
24
 
# File 'lib/archaeo/bulk_downloader.rb', line 17

def initialize(client: HttpClient.new, output_dir: "archive",
               cdx_api: nil, concurrency: 1, on_error: nil)
  @client = client
  @output_dir = output_dir
  @cdx_api = cdx_api
  @concurrency = [1, concurrency.to_i].max
  @on_error = on_error
end

Instance Method Details

#download(url, from: nil, to: nil, resume: false, dry_run: false, &block) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/archaeo/bulk_downloader.rb', line 26

def download(url, from: nil, to: nil, resume: false,
             dry_run: false, &block)
  start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
  url = UrlNormalizer.normalize(url)
  FileUtils.mkdir_p(@output_dir) unless dry_run

  snapshots = fetch_snapshots(url, from: from, to: to)
  downloaded, skipped, bytes, failed =
    run_download(snapshots, resume, dry_run, block)

  build_summary(start_time, snapshots.size, downloaded,
                skipped, bytes, failed: failed)
end