Class: WebScrapingAI::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/webscraping_ai/client.rb

Constant Summary collapse

PROXY_TYPES = 
%w[datacenter residential stealth].freeze
COUNTRIES = 
%w[us gb de it fr ca es ru jp kr in hk tr].freeze
DEVICES = 
%w[desktop mobile tablet].freeze
TEXT_FORMATS = 
%w[plain xml json].freeze
FORMATS = 
%w[json text].freeze
PAGE_FETCH_OPTIONS = 
%i[
  headers timeout js js_timeout wait_for proxy country
  custom_proxy device error_on_404 error_on_redirect js_script
].freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(api_key: nil, base_url: nil, timeout: nil, open_timeout: nil, adapter: nil, user_agent: nil) ⇒ Client

Returns a new instance of Client.

Raises:



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/webscraping_ai/client.rb', line 19

def initialize(api_key: nil, base_url: nil, timeout: nil, open_timeout: nil, adapter: nil, user_agent: nil)
  global = WebScrapingAI.configuration
  @configuration = Configuration.new.tap do |c|
    c.api_key = api_key || global.api_key
    c.base_url = base_url || global.base_url
    c.timeout = timeout || global.timeout
    c.open_timeout = open_timeout || global.open_timeout
    c.adapter = adapter || global.adapter
    c.user_agent = user_agent || global.user_agent
  end

  return unless @configuration.api_key.nil? || @configuration.api_key.to_s.empty?

  raise ConfigurationError,
        "api_key is required (pass api_key: or set WebScrapingAI.configure { |c| c.api_key = ... })"
end

Instance Attribute Details

#configuration ⇒ Object (readonly)

Returns the value of attribute configuration.



17
18
19
 
# File 'lib/webscraping_ai/client.rb', line 17

def configuration
  @configuration
end

Instance Method Details

#account ⇒ Object

GET /account — returns Hash with remaining_api_calls, resets_at, remaining_concurrency, email.



70
71
72
 
# File 'lib/webscraping_ai/client.rb', line 70

def 
  get("/account")
end

#fields(url, fields:, **opts) ⇒ Object

GET /ai/fields — extracts the named fields from the page. fields is a Hash of { field_name => description }. Returns a Hash.



44
45
46
 
# File 'lib/webscraping_ai/client.rb', line 44

def fields(url, fields:, **opts)
  get("/ai/fields", url: url, fields: fields, **opts.slice(*PAGE_FETCH_OPTIONS))
end

#html(url, **opts) ⇒ Object

GET /html — returns the full page HTML as a String.



49
50
51
 
# File 'lib/webscraping_ai/client.rb', line 49

def html(url, **opts)
  get("/html", url: url, **opts.slice(*PAGE_FETCH_OPTIONS, :return_script_result, :format))
end

#question(url, question:, **opts) ⇒ Object

GET /ai/question — returns the LLM's answer about the page. Returns a String by default, or a Hash when format: "json".



38
39
40
 
# File 'lib/webscraping_ai/client.rb', line 38

def question(url, question:, **opts)
  get("/ai/question", url: url, question: question, **opts.slice(*PAGE_FETCH_OPTIONS, :format))
end

#selected(url, selector: nil, **opts) ⇒ Object

GET /selected — returns HTML of the element matching selector as a String.



60
61
62
 
# File 'lib/webscraping_ai/client.rb', line 60

def selected(url, selector: nil, **opts)
  get("/selected", url: url, selector: selector, **opts.slice(*PAGE_FETCH_OPTIONS, :format))
end

#selected_multiple(url, selectors:, **opts) ⇒ Object

GET /selected-multiple — returns an Array of HTML strings, one per selector.



65
66
67
 
# File 'lib/webscraping_ai/client.rb', line 65

def selected_multiple(url, selectors:, **opts)
  get("/selected-multiple", url: url, selectors: Array(selectors), **opts.slice(*PAGE_FETCH_OPTIONS))
end

#text(url, **opts) ⇒ Object

GET /text — returns the visible text content of the page. Returns a String when text_format is "plain"/"xml" (default), or a Hash when text_format: "json".



55
56
57
 
# File 'lib/webscraping_ai/client.rb', line 55

def text(url, **opts)
  get("/text", url: url, **opts.slice(*PAGE_FETCH_OPTIONS, :text_format, :return_links))
end