Module: ScrapeUnblocker

Defined in:
lib/scrapeunblocker.rb,
lib/scrapeunblocker/client.rb,
lib/scrapeunblocker/errors.rb,
lib/scrapeunblocker/version.rb,
lib/scrapeunblocker/skyscanner.rb,
lib/scrapeunblocker/parsed_page.rb

Overview

Official Ruby client for the ScrapeUnblocker web scraping API.

require "scrapeunblocker"

su = ScrapeUnblocker::Client.new  # reads SCRAPEUNBLOCKER_KEY
html = su.get_page_source("https://example.com")
product = su.get_parsed("https://www.amazon.com/dp/B08N5WRWNW")

Defined Under Namespace

Classes: APIError, AuthenticationError, BlockedError, Client, ConnectionError, Error, InvalidRequestError, PageResult, ParsedPage, RateLimitError, ServerError, Skyscanner, TimeoutError, UpstreamOutageError

Constant Summary collapse

VERSION = 
"0.1.0"

Class Method Summary collapse

Class Method Details

.error_for_status(status, body) ⇒ Object

Build a typed error from an HTTP status code and response body.



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/scrapeunblocker/errors.rb', line 44

def self.error_for_status(status, body)
  snippet = (body || "").strip.gsub(/\s+/, " ")
  snippet = "#{snippet[0, 200]}..." if snippet.length > 200
  base = {
    400 => "Invalid request (bad URL or unsupported scheme)",
    401 => "Authentication failed - check your API key",
    403 => "Target blocked by bot protection on every bypass path",
    429 => "Rate limited - too many requests",
    503 => "Upstream origin returned a server-side outage page"
  }.fetch(status, "API returned HTTP #{status}")
  message = snippet.empty? ? base : "#{base}: #{snippet}"

  klass =
    case status
    when 400 then InvalidRequestError
    when 401 then AuthenticationError
    when 403 then BlockedError
    when 429 then RateLimitError
    when 503 then UpstreamOutageError
    else status >= 500 ? ServerError : APIError
    end

  klass.new(message, status_code: status, body: body)
end