Module: ArchiveAPI

Included in:
WaybackMachineDownloader
Defined in:
lib/wayback_machine_downloader/archive_api.rb

Defined Under Namespace

Classes: RateLimitError

Constant Summary collapse

DEFAULT_RATE_LIMIT_COOLDOWN = 
30.0
DEFAULT_CDX_INTERVAL =

1 request every 2.5 seconds

2.5

Class Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Class Attribute Details

.cdx_interval ⇒ Object

Returns the value of attribute cdx_interval.



25
26
27
 
# File 'lib/wayback_machine_downloader/archive_api.rb', line 25

def cdx_interval
  @cdx_interval
end

Class Method Details

.extend_cdx_cooldown(seconds) ⇒ Object

extend the cooldown period for CDX requests, e.g., after receiving a 429 response



45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/wayback_machine_downloader/archive_api.rb', line 45

def extend_cdx_cooldown(seconds)
  seconds = seconds.to_f
  return if seconds <= 0

  @cdx_mutex.synchronize do
    now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    candidate = now + seconds
    @next_allowed_cdx_at = candidate if candidate > @next_allowed_cdx_at
    @cdx_cv.broadcast
  end
end

.pace_cdx_request ⇒ Object

pace CDX requests to avoid exceeding the rate limit



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/wayback_machine_downloader/archive_api.rb', line 28

def pace_cdx_request
  @cdx_mutex.synchronize do
    loop do
      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
      if now < @next_allowed_cdx_at
        wait_time = @next_allowed_cdx_at - now
        @cdx_cv.wait(@cdx_mutex, wait_time)
      else
        interval = @cdx_interval || DEFAULT_CDX_INTERVAL
        @next_allowed_cdx_at = now + interval
        return
      end
    end
  end
end

.reset_cdx_limiter ⇒ Object 



57
58
59
60
61
62
 
# File 'lib/wayback_machine_downloader/archive_api.rb', line 57

def reset_cdx_limiter
  @cdx_mutex.synchronize do
    @next_allowed_cdx_at = 0.0
    @cdx_cv.broadcast
  end
end

Instance Method Details

#get_raw_list_from_api(url, page_index, http) ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
 
# File 'lib/wayback_machine_downloader/archive_api.rb', line 65

def get_raw_list_from_api(url, page_index, http)
  # Automatically append /* for host-only URLs
  # This is a workaround for an issue with the API and *some* domains.
  # See https://github.com/StrawberryMaster/wayback-machine-downloader/issues/6
  # But don't do this when exact_url flag is set, and never append twice
  normalized_url = url.to_s.strip
  
  # strip protocol for CDX query
  clean_url = normalized_url.sub(%r{\Ahttps?://}i, '')
  # ensure wildcard/matchType for domain-wide crawling
  match_type = nil
  unless @exact_url || clean_url.include?('*')
    if clean_url.end_with?('/')
      clean_url = "#{clean_url}*"
    elsif !clean_url.include?('/')
      match_type = "prefix"
    else
      clean_url = "#{clean_url}/*"
    end
  end

  request_url = URI("https://web.archive.org/cdx/search/cdx")
  params = [["output", "json"], ["url", clean_url]] + parameters_for_api(page_index)
  params << ["matchType", match_type] if match_type
  request_url.query = URI.encode_www_form(params)

  retries = 0
  max_retries = (@max_retries || 3)
  base_delay = 2

  begin
    # acquire slot from the process-wide proactive pacer before sending request
    ArchiveAPI.pace_cdx_request

    if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
      response = http.get(request_url)
      raise response.error if response.is_a?(HTTPX::ErrorResponse)

      code = response.status
      body = response.body.to_s.strip
    else
      request = Net::HTTP::Get.new(request_url)
      request["User-Agent"] = "wmd-straw/#{WaybackMachineDownloader::VERSION rescue '2.4.8'}"
      request["Connection"] = "keep-alive"
      request["Accept-Encoding"] = "gzip, deflate"
      response = http.request(request)
      code = response.code.to_i
      body = decompress_body(response)
    end

    case code
    when 200
      return [] if body.empty?
      begin
        json = JSON.parse(body)
        # check if the response contains the header ["timestamp", "original"]
        json.shift if json.first == ["timestamp", "original"]
        json
      rescue JSON::ParserError => e
        raise "Malformed JSON response: #{e.message}"
      end
    when 400
      # CDX API occasionally returns 400 when page index exceeds total available pages (that is, end of pagination)
      return []
    when 429
      retry_after = retry_after_seconds(response)
      raise RateLimitError.new(
        "Server error 429: #{response.respond_to?(:message) ? response.message : 'Too Many Requests'}",
        retry_after
      )
    when 500, 502, 503, 504
      raise "Server error #{code}: #{response.respond_to?(:message) ? response.message : ''}"
    else
      raise "Unexpected API response #{code} for #{url}"
    end
  rescue Net::ReadTimeout, Net::OpenTimeout, StandardError => e
    if retries < max_retries
      retries += 1
      jitter = rand(0.0..1.0)

      if e.is_a?(RateLimitError)
        # if the server provided a Retry-After header, use that; otherwise, use an exponential backoff with a minimum cooldown
        fallback = [DEFAULT_RATE_LIMIT_COOLDOWN, base_delay * (2 ** (retries - 1))].max
        cooldown = (e.retry_after || fallback) + (e.retry_after ? 0 : jitter)
        ArchiveAPI.extend_cdx_cooldown(cooldown)

        warn "Wayback CDX API rate limited (429) for #{url}. " \
             "Pausing CDX requests for #{cooldown.round(2)}s " \
             "(attempt #{retries}/#{max_retries})..."
      else
        sleep_time = (base_delay * (2 ** (retries - 1))) + jitter
        warn "Error talking to Wayback CDX API (#{e.class}: #{e.message}) for #{url}. " \
             "Retrying in #{sleep_time.round(2)}s (attempt #{retries}/#{max_retries})..."
        sleep(sleep_time)
      end

      retry
    else
      warn "Giving up on Wayback CDX API for #{url} after #{max_retries} attempts. (Last error: #{e.message})"
      raise
    end
  end
end

#parameters_for_api(page_index) ⇒ Object 



169
170
171
172
173
174
175
176
177
 
# File 'lib/wayback_machine_downloader/archive_api.rb', line 169

def parameters_for_api(page_index)
  parameters = [["fl", "timestamp,original"], ["gzip", "true"]]
  parameters.push(["collapse", "digest"]) unless @keep_duplicates || @all_timestamps
  parameters.push(["filter", "statuscode:2..|30[12378]"]) unless @all
  parameters.push(["from", @from_timestamp.to_s]) if @from_timestamp && @from_timestamp != 0
  parameters.push(["to", @to_timestamp.to_s]) if @to_timestamp && @to_timestamp != 0
  parameters.push(["page", page_index.to_s]) if page_index && page_index > 0
  parameters
end