Class: ScrapeUnblocker::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/scrapeunblocker/client.rb

Overview

Client for the ScrapeUnblocker API.

su = ScrapeUnblocker::Client.new(api_key: "YOUR_API_KEY")
html = su.get_page_source("https://example.com")

Constant Summary collapse

DEFAULT_BASE_URL = 
"https://api.scrapeunblocker.com"
API_KEY_HEADER = 
"x-scrapeunblocker-key"
RETRYABLE = 
[429, 502, 503, 504].freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(api_key: nil, base_url: DEFAULT_BASE_URL, timeout: 180, max_retries: 2, transport: nil) ⇒ Client

Returns a new instance of Client.



25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/scrapeunblocker/client.rb', line 25

def initialize(api_key: nil, base_url: DEFAULT_BASE_URL, timeout: 180, max_retries: 2, transport: nil)
  @api_key = api_key || ENV["SCRAPEUNBLOCKER_KEY"]
  if @api_key.nil? || @api_key.empty?
    raise Error, "No API key provided. Pass api_key: or set the " \
                 "SCRAPEUNBLOCKER_KEY environment variable. Get your key " \
                 "at https://app.scrapeunblocker.com"
  end
  @base_url = base_url.sub(%r{/+\z}, "")
  @timeout = timeout
  @max_retries = max_retries
  @transport = transport || method(:net_http_transport)
  @skyscanner = Skyscanner.new(self)
end

Instance Attribute Details

#skyscanner ⇒ Skyscanner (readonly)

Returns the Skyscanner plugin endpoints.

Returns:

  • (Skyscanner) —

    the Skyscanner plugin endpoints



23
24
25
 
# File 'lib/scrapeunblocker/client.rb', line 23

def skyscanner
  @skyscanner
end

Instance Method Details

#get_image(url, proxy_country: nil) ⇒ Object

Fetch an image URL through the bypass chain and return its raw bytes.



74
75
76
 
# File 'lib/scrapeunblocker/client.rb', line 74

def get_image(url, proxy_country: nil)
  request("/getImage", url: url, proxy_country: proxy_country)[:body]
end

#get_page_source(url, proxy_country: nil, time_sleep: nil, method: nil, value: nil, method_timeout: nil) ⇒ Object

Fetch a URL and return the fully rendered HTML.



40
41
42
43
44
 
# File 'lib/scrapeunblocker/client.rb', line 40

def get_page_source(url, proxy_country: nil, time_sleep: nil, method: nil, value: nil, method_timeout: nil)
  request("/getPageSource",
          url: url, proxy_country: proxy_country, time_sleep: time_sleep,
          method: method, value: value, method_timeout: method_timeout)[:body]
end

#get_page_with_cookies(url, proxy_country: nil, time_sleep: nil) ⇒ Object

Fetch a URL and also return the cookies and proxy that served it.



57
58
59
60
61
62
 
# File 'lib/scrapeunblocker/client.rb', line 57

def get_page_with_cookies(url, proxy_country: nil, time_sleep: nil)
  body = request("/getPageSource",
                 url: url, get_cookies: true, proxy_country: proxy_country,
                 time_sleep: time_sleep)[:body]
  PageResult.from_hash(JSON.parse(body))
end

#get_parsed(url, proxy_country: nil, time_sleep: nil, refresh_rules: false, rules_hint: nil) ⇒ Object

Fetch a URL and return structured JSON instead of HTML.



47
48
49
50
51
52
53
54
 
# File 'lib/scrapeunblocker/client.rb', line 47

def get_parsed(url, proxy_country: nil, time_sleep: nil, refresh_rules: false, rules_hint: nil)
  body = request("/getPageSource",
                 url: url, parsed_data: true, proxy_country: proxy_country,
                 time_sleep: time_sleep,
                 refresh_rules: (refresh_rules ? true : nil),
                 rules_hint: rules_hint)[:body]
  ParsedPage.from_hash(JSON.parse(body))
end

#post_json(path, params) ⇒ Object

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future. 



79
80
81
 
# File 'lib/scrapeunblocker/client.rb', line 79

def post_json(path, params)
  JSON.parse(request(path, params)[:body])
end

#serp(keyword, proxy_country: nil, pages_to_check: 1, wait_after_load: 0, captcha_pause: 0) ⇒ Object

Run a Google search and return the parsed SERP as a Hash.



65
66
67
68
69
70
71
 
# File 'lib/scrapeunblocker/client.rb', line 65

def serp(keyword, proxy_country: nil, pages_to_check: 1, wait_after_load: 0, captcha_pause: 0)
  post_json("/serpApi",
            keyword: keyword, proxy_country: proxy_country,
            pages_to_check: pages_to_check,
            wait_after_load: (wait_after_load.zero? ? nil : wait_after_load),
            captcha_pause: (captcha_pause.zero? ? nil : captcha_pause))
end