Class: Webpipe::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/webpipe/client.rb

Overview

The WebPipe API client.

client = Webpipe::Client.new # reads WEBPIPE_API_KEY
doc = client.scrape("https://example.com", formats: ["markdown"])
puts doc.markdown

Constant Summary collapse

DEFAULT_BASE_URL = 
"https://api.web2json.ai"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(api_key: nil, base_url: nil, timeout: 60, max_retries: 2) ⇒ Client

Returns a new instance of Client.



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/webpipe/client.rb', line 14

def initialize(api_key: nil, base_url: nil, timeout: 60, max_retries: 2)
  api_key ||= ENV["WEBPIPE_API_KEY"]
  if api_key.nil? || api_key.empty?
    raise ArgumentError,
          "Missing API key. Pass api_key: or set the WEBPIPE_API_KEY environment " \
          "variable. Create one at https://webpipe.ai/api-keys.html"
  end

  base_url ||= ENV["WEBPIPE_API_URL"] || DEFAULT_BASE_URL
  @http = HttpClient.new(
    base_url: base_url.sub(%r{/+$}, ""),
    api_key: api_key,
    timeout: timeout,
    max_retries: max_retries,
    user_agent: "webpipe-sdk-ruby/#{Webpipe::VERSION}"
  )
  @browser = BrowserResource.new(@http)
end

Instance Attribute Details

#browser ⇒ Object (readonly)

Returns the value of attribute browser.



12
13
14
 
# File 'lib/webpipe/client.rb', line 12

def browser
  @browser
end

Instance Method Details

#crawl(url, poll_interval: 2, timeout: nil, **options) ⇒ Object

Submit a crawl job and poll until it completes.

Yields the current CrawlJob after each poll when a block is given — the idiomatic Ruby way to observe a long crawl (progress bars, logging).

Raises CrawlError if the job fails, PollTimeoutError if timeout is exceeded. Other keywords are the same as start_crawl.



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/webpipe/client.rb', line 74

def crawl(url, poll_interval: 2, timeout: nil, **options)
  job = start_crawl(url, **options)
  deadline = timeout && Time.now + timeout
  loop do
    status = get_crawl_status(job.id)
    return status if status.status == CrawlJob::STATUS_COMPLETED
    raise CrawlError, "Crawl job #{job.id} failed: #{status.errors}" if status.status == CrawlJob::STATUS_FAILED
    yield status if block_given?
    if deadline && Time.now >= deadline
      raise PollTimeoutError, "Crawl job #{job.id} did not complete within #{timeout}s"
    end

    sleep poll_interval
  end
end

#get_crawl_status(job_id) ⇒ Object

Fetch the current status and partial results of a crawl job.



63
64
65
 
# File 'lib/webpipe/client.rb', line 63

def get_crawl_status(job_id)
  CrawlJob.from_hash(@http.get("/api/v1/crawl/#{job_id}"))
end

#map(url, limit: nil, search: nil, sitemap: nil, same_domain: nil, use_proxy: nil, proxy: nil, cookie: nil) ⇒ Object

Discover URLs of a website (robots.txt → sitemap → in-page links).



40
41
42
43
44
 
# File 'lib/webpipe/client.rb', line 40

def map(url, limit: nil, search: nil, sitemap: nil, same_domain: nil,
        use_proxy: nil, proxy: nil, cookie: nil)
  body = drop_nil(url:, limit:, search:, sitemap:, same_domain:, use_proxy:, proxy:, cookie:)
  (@http.post("/api/v1/map", body)["links"] || []).map { |l| MapLink.from_hash(l) }
end

#scrape(url, formats: nil, json_schema: nil, use_proxy: nil, proxy: nil, cookie: nil) ⇒ Object

Scrape a single page into markdown/html/links/metadata/json.



34
35
36
37
 
# File 'lib/webpipe/client.rb', line 34

def scrape(url, formats: nil, json_schema: nil, use_proxy: nil, proxy: nil, cookie: nil)
  body = drop_nil(url:, formats:, json_schema:, use_proxy:, proxy:, cookie:)
  Document.from_hash(@http.post("/api/v1/scrape", body)["data"])
end

#start_crawl(url, limit: nil, max_discovery_depth: nil, include_paths: nil, exclude_paths: nil, allow_subdomains: nil, allow_external_links: nil, crawl_entire_domain: nil, ignore_query_params: nil, sitemap: nil, delay: nil, max_concurrency: nil, scrape_options: nil, use_proxy: nil, proxy: nil, cookie: nil) ⇒ Object

Submit an async crawl job and return its handle immediately.



47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/webpipe/client.rb', line 47

def start_crawl(url, limit: nil, max_discovery_depth: nil, include_paths: nil,
                exclude_paths: nil, allow_subdomains: nil, allow_external_links: nil,
                crawl_entire_domain: nil, ignore_query_params: nil, sitemap: nil,
                delay: nil, max_concurrency: nil, scrape_options: nil,
                use_proxy: nil, proxy: nil, cookie: nil)
  body = drop_nil(
    url:, limit:, max_discovery_depth:, include_paths:, exclude_paths:,
    allow_subdomains:, allow_external_links:, crawl_entire_domain:,
    ignore_query_params:, sitemap:, delay:, max_concurrency:,
    scrape_options:, use_proxy:, proxy:, cookie:
  )
  res = @http.post("/api/v1/crawl", body)
  CrawlJobStart.new(res.fetch("id"), res["url"])
end