Class: Archaeo::BulkDownloader

Inherits:
Object
  • Object
show all
Defined in:
lib/archaeo/bulk_downloader.rb

Overview

Downloads all archived snapshots of a URL with resume support.

Queries the CDX API for matching snapshots, fetches each page, and saves content to disk. Progress is tracked in a state file for interrupted download recovery.

Instance Method Summary collapse

Constructor Details

#initialize(client: HttpClient.new, output_dir: "archive", cdx_api: nil, concurrency: 1) ⇒ BulkDownloader

Returns a new instance of BulkDownloader.



17
18
19
20
21
22
23
 
# File 'lib/archaeo/bulk_downloader.rb', line 17

def initialize(client: HttpClient.new, output_dir: "archive",
               cdx_api: nil, concurrency: 1)
  @client = client
  @output_dir = output_dir
  @cdx_api = cdx_api
  @concurrency = [1, concurrency.to_i].max
end

Instance Method Details

#download(url, from: nil, to: nil, resume: false, dry_run: false, &block) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/archaeo/bulk_downloader.rb', line 25

def download(url, from: nil, to: nil, resume: false,
             dry_run: false, &block)
  start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
  url = UrlNormalizer.normalize(url)
  FileUtils.mkdir_p(@output_dir) unless dry_run

  snapshots = fetch_snapshots(url, from: from, to: to)
  downloaded, skipped, bytes =
    run_download(snapshots, resume, dry_run, block)

  build_summary(start_time, snapshots.size, downloaded, skipped, bytes)
end