Class: Berrycrawl::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/Berrycrawl/client.rb

Instance Method Summary collapse

Constructor Details

#initialize(base_url: nil, token: ENV.fetch("BERRYCRAWL_API_KEY", nil), max_retries: 2) ⇒ void

Parameters:

  • base_url (String, nil) (defaults to: nil)
  • token (String) (defaults to: ENV.fetch("BERRYCRAWL_API_KEY", nil))
  • max_retries (Integer) (defaults to: 2) 


237
238
239
240
241
242
243
244
245
246
 
# File 'lib/Berrycrawl/client.rb', line 237

def initialize(base_url: nil, token: ENV.fetch("BERRYCRAWL_API_KEY", nil), max_retries: 2)
  @raw_client = Berrycrawl::Internal::Http::RawClient.new(
    base_url: base_url || Berrycrawl::Environment::PRODUCTION,
    headers: {
      "X-Fern-Language" => "Ruby",
      Authorization: "Bearer #{token}"
    },
    max_retries: max_retries
  )
end

Instance Method Details

#account ⇒ Berrycrawl::Account::Client

Returns:



249
250
251
 
# File 'lib/Berrycrawl/client.rb', line 249

def 
  @account ||= Berrycrawl::Account::Client.new(client: @raw_client)
end

#brand ⇒ Berrycrawl::Brand::Client

Returns:



254
255
256
 
# File 'lib/Berrycrawl/client.rb', line 254

def brand
  @brand ||= Berrycrawl::Brand::Client.new(client: @raw_client)
end

#crawl(request_options: {}, **params) ⇒ Berrycrawl::Types::JobCreatedResponse

Parameters:

Options Hash (request_options:):

  • :base_url (String)
  • :additional_headers (Hash{String => Object})
  • :additional_query_parameters (Hash{String => Object})
  • :additional_body_parameters (Hash{String => Object})
  • :timeout_in_seconds (Integer)

Returns:



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/Berrycrawl/client.rb', line 14

def crawl(request_options: {}, **params)
  params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
  request = Berrycrawl::Internal::JSON::Request.new(
    base_url: request_options[:base_url],
    method: "POST",
    path: "crawl",
    body: Berrycrawl::Types::CrawlDto.new(params).to_h,
    request_options: request_options
  )
  begin
    response = @client.send(request)
  rescue Net::HTTPRequestTimeout
    raise Berrycrawl::Errors::TimeoutError
  end
  code = response.code.to_i
  if code.between?(200, 299)
    Berrycrawl::Types::JobCreatedResponse.load(response.body)
  else
    error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
    raise error_class.new(response.body, code: code)
  end
end

#extract(request_options: {}, **params) ⇒ Berrycrawl::Types::JobCreatedResponse

Parameters:

Options Hash (request_options:):

  • :base_url (String)
  • :additional_headers (Hash{String => Object})
  • :additional_query_parameters (Hash{String => Object})
  • :additional_body_parameters (Hash{String => Object})
  • :timeout_in_seconds (Integer)

Returns:



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/Berrycrawl/client.rb', line 46

def extract(request_options: {}, **params)
  params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
  request = Berrycrawl::Internal::JSON::Request.new(
    base_url: request_options[:base_url],
    method: "POST",
    path: "extract",
    body: Berrycrawl::Types::ExtractDto.new(params).to_h,
    request_options: request_options
  )
  begin
    response = @client.send(request)
  rescue Net::HTTPRequestTimeout
    raise Berrycrawl::Errors::TimeoutError
  end
  code = response.code.to_i
  if code.between?(200, 299)
    Berrycrawl::Types::JobCreatedResponse.load(response.body)
  else
    error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
    raise error_class.new(response.body, code: code)
  end
end

#jobs ⇒ Berrycrawl::Jobs::Client

Returns:



259
260
261
 
# File 'lib/Berrycrawl/client.rb', line 259

def jobs
  @jobs ||= Berrycrawl::Jobs::Client.new(client: @raw_client)
end

#map(request_options: {}, **params) ⇒ Berrycrawl::Types::MapResponse

Parameters:

Options Hash (request_options:):

  • :base_url (String)
  • :additional_headers (Hash{String => Object})
  • :additional_query_parameters (Hash{String => Object})
  • :additional_body_parameters (Hash{String => Object})
  • :timeout_in_seconds (Integer)

Returns:



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/Berrycrawl/client.rb', line 78

def map(request_options: {}, **params)
  params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
  request = Berrycrawl::Internal::JSON::Request.new(
    base_url: request_options[:base_url],
    method: "POST",
    path: "map",
    body: Berrycrawl::Types::MapDto.new(params).to_h,
    request_options: request_options
  )
  begin
    response = @client.send(request)
  rescue Net::HTTPRequestTimeout
    raise Berrycrawl::Errors::TimeoutError
  end
  code = response.code.to_i
  if code.between?(200, 299)
    Berrycrawl::Types::MapResponse.load(response.body)
  else
    error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
    raise error_class.new(response.body, code: code)
  end
end

#parse(request_options: {}, **params) ⇒ Berrycrawl::Types::ScrapeResponse

Parameters:

Options Hash (request_options:):

  • :base_url (String)
  • :additional_headers (Hash{String => Object})
  • :additional_query_parameters (Hash{String => Object})
  • :additional_body_parameters (Hash{String => Object})
  • :timeout_in_seconds (Integer)

Returns:



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/Berrycrawl/client.rb', line 110

def parse(request_options: {}, **params)
  params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
  request = Berrycrawl::Internal::JSON::Request.new(
    base_url: request_options[:base_url],
    method: "POST",
    path: "parse",
    body: Berrycrawl::Types::ParseDto.new(params).to_h,
    request_options: request_options
  )
  begin
    response = @client.send(request)
  rescue Net::HTTPRequestTimeout
    raise Berrycrawl::Errors::TimeoutError
  end
  code = response.code.to_i
  if code.between?(200, 299)
    Berrycrawl::Types::ScrapeResponse.load(response.body)
  else
    error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
    raise error_class.new(response.body, code: code)
  end
end

#scrape(request_options: {}, **params) ⇒ Berrycrawl::Types::ScrapeResponse

Parameters:

Options Hash (request_options:):

  • :base_url (String)
  • :additional_headers (Hash{String => Object})
  • :additional_query_parameters (Hash{String => Object})
  • :additional_body_parameters (Hash{String => Object})
  • :timeout_in_seconds (Integer)

Returns:



142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'lib/Berrycrawl/client.rb', line 142

def scrape(request_options: {}, **params)
  params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
  request = Berrycrawl::Internal::JSON::Request.new(
    base_url: request_options[:base_url],
    method: "POST",
    path: "scrape",
    body: Berrycrawl::Types::ScrapeDto.new(params).to_h,
    request_options: request_options
  )
  begin
    response = @client.send(request)
  rescue Net::HTTPRequestTimeout
    raise Berrycrawl::Errors::TimeoutError
  end
  code = response.code.to_i
  if code.between?(200, 299)
    Berrycrawl::Types::ScrapeResponse.load(response.body)
  else
    error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
    raise error_class.new(response.body, code: code)
  end
end

#screenshot(request_options: {}, **params) ⇒ Berrycrawl::Types::ScrapeResponse

Cookie banners, blocking overlays, chat widgets, and lazy loading are handled by default. Every cleanup pass can be controlled per request.

Parameters:

Options Hash (request_options:):

  • :base_url (String)
  • :additional_headers (Hash{String => Object})
  • :additional_query_parameters (Hash{String => Object})
  • :additional_body_parameters (Hash{String => Object})
  • :timeout_in_seconds (Integer)

Returns:



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/Berrycrawl/client.rb', line 177

def screenshot(request_options: {}, **params)
  params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
  request = Berrycrawl::Internal::JSON::Request.new(
    base_url: request_options[:base_url],
    method: "POST",
    path: "screenshot",
    body: Berrycrawl::Types::ScreenshotDto.new(params).to_h,
    request_options: request_options
  )
  begin
    response = @client.send(request)
  rescue Net::HTTPRequestTimeout
    raise Berrycrawl::Errors::TimeoutError
  end
  code = response.code.to_i
  if code.between?(200, 299)
    Berrycrawl::Types::ScrapeResponse.load(response.body)
  else
    error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
    raise error_class.new(response.body, code: code)
  end
end

#search(request_options: {}, **params) ⇒ Berrycrawl::Types::SearchResponse

Parameters:

Options Hash (request_options:):

  • :base_url (String)
  • :additional_headers (Hash{String => Object})
  • :additional_query_parameters (Hash{String => Object})
  • :additional_body_parameters (Hash{String => Object})
  • :timeout_in_seconds (Integer)

Returns:



209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
 
# File 'lib/Berrycrawl/client.rb', line 209

def search(request_options: {}, **params)
  params = Berrycrawl::Internal::Types::Utils.normalize_keys(params)
  request = Berrycrawl::Internal::JSON::Request.new(
    base_url: request_options[:base_url],
    method: "POST",
    path: "search",
    body: Berrycrawl::Types::SearchDto.new(params).to_h,
    request_options: request_options
  )
  begin
    response = @client.send(request)
  rescue Net::HTTPRequestTimeout
    raise Berrycrawl::Errors::TimeoutError
  end
  code = response.code.to_i
  if code.between?(200, 299)
    Berrycrawl::Types::SearchResponse.load(response.body)
  else
    error_class = Berrycrawl::Errors::ResponseError.subclass_for_code(code)
    raise error_class.new(response.body, code: code)
  end
end

#webhooks ⇒ Berrycrawl::Webhooks::Client

Returns:



264
265
266
 
# File 'lib/Berrycrawl/client.rb', line 264

def webhooks
  @webhooks ||= Berrycrawl::Webhooks::Client.new(client: @raw_client)
end