Module: ScrapeUnblocker

Defined in:
lib/scrapeunblocker.rb,
lib/scrapeunblocker/client.rb,
lib/scrapeunblocker/errors.rb,
lib/scrapeunblocker/version.rb,
lib/scrapeunblocker/skyscanner.rb,
lib/scrapeunblocker/parsed_page.rb

Overview

Official Ruby client for the ScrapeUnblocker web scraping API.

require "scrapeunblocker"

su = ScrapeUnblocker::Client.new  # reads SCRAPEUNBLOCKER_KEY
html = su.get_page_source("https://example.com")
product = su.get_parsed("https://www.amazon.com/dp/B08N5WRWNW")

Defined Under Namespace

Classes: APIError, AuthenticationError, BlockedError, BrowserTimeoutError, Client, ConnectionError, CreditLimitExceededError, Error, InvalidRequestError, NoSubscriptionError, NotFoundError, PageResult, ParsedPage, PaymentFailedError, PaymentRequiredError, QuotaExceededError, RateLimitError, ServerError, Skyscanner, TimeoutError, UnsupportedContentError, UpstreamOutageError, ValidationError

Constant Summary collapse

VERSION = 
"0.1.7"

Class Method Summary collapse

Class Method Details

.error_for_status(status, body) ⇒ Object

Build a typed error from an HTTP status code and response body.



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/scrapeunblocker/errors.rb', line 157

def self.error_for_status(status, body)
  snippet = (body || "").strip.gsub(/\s+/, " ")
  snippet = "#{snippet[0, 200]}..." if snippet.length > 200
  base = BASE_MESSAGES.fetch(status, "API returned HTTP #{status}")
  message = snippet.empty? ? base : "#{base}: #{snippet}"

  klass =
    case status
    when 400 then InvalidRequestError
    when 401 then auth_error_class(body)
    when 402 then billing_error_class(body)
    when 403 then BlockedError
    when 404 then NotFoundError
    when 408 then BrowserTimeoutError
    when 415 then UnsupportedContentError
    when 422 then ValidationError
    when 429 then RateLimitError
    when 503 then UpstreamOutageError
    else status >= 500 ? ServerError : APIError
    end

  klass.new(message, status_code: status, body: body)
end