Class: Crawlbase::StorageAPI

Inherits:
Object
  • Object
show all
Defined in:
lib/crawlbase/storage_api.rb

Constant Summary collapse

INVALID_TOKEN = 
'Token is required'
INVALID_RID = 
'RID is required'
INVALID_RID_ARRAY = 
'One or more RIDs are required'
INVALID_URL_OR_RID = 
'Either URL or RID is required'
BASE_URL = 
'https://api.crawlbase.com/storage'

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ StorageAPI

Returns a new instance of StorageAPI.

Raises:



20
21
22
23
24
 
# File 'lib/crawlbase/storage_api.rb', line 20

def initialize(options = {})
  raise INVALID_TOKEN if options[:token].nil? || options[:token].empty?

  @token = options[:token]
end

Instance Attribute Details

#body ⇒ Object (readonly)

Returns the value of attribute body.



9
10
11
 
# File 'lib/crawlbase/storage_api.rb', line 9

def body
  @body
end

#cb_status ⇒ Object (readonly)

Crawlbase status from the response (preferred name for former pc_status).



12
13
14
 
# File 'lib/crawlbase/storage_api.rb', line 12

def cb_status
  @cb_status
end

#original_status ⇒ Object (readonly)

Returns the value of attribute original_status.



9
10
11
 
# File 'lib/crawlbase/storage_api.rb', line 9

def original_status
  @original_status
end

#rid ⇒ Object (readonly)

Returns the value of attribute rid.



9
10
11
 
# File 'lib/crawlbase/storage_api.rb', line 9

def rid
  @rid
end

#status_code ⇒ Object (readonly)

Returns the value of attribute status_code.



9
10
11
 
# File 'lib/crawlbase/storage_api.rb', line 9

def status_code
  @status_code
end

#stored_at ⇒ Object (readonly)

Returns the value of attribute stored_at.



9
10
11
 
# File 'lib/crawlbase/storage_api.rb', line 9

def stored_at
  @stored_at
end

#token ⇒ Object (readonly)

Returns the value of attribute token.



9
10
11
 
# File 'lib/crawlbase/storage_api.rb', line 9

def token
  @token
end

#url ⇒ Object (readonly)

Returns the value of attribute url.



9
10
11
 
# File 'lib/crawlbase/storage_api.rb', line 9

def url
  @url
end

Instance Method Details

#bulk(rids_array = []) ⇒ Object

Raises:



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/crawlbase/storage_api.rb', line 71

def bulk(rids_array = [])
  raise INVALID_RID_ARRAY if rids_array.empty?

  uri = URI("#{BASE_URL}/bulk")
  uri.query = URI.encode_www_form(token: token)
  http = Net::HTTP.new(uri.host)
  request = Net::HTTP::Post.new(uri.request_uri, { 'Content-Type': 'application/json' })
  request.body = { rids: rids_array }.to_json
  response = http.request(request)

  @body = JSON.parse(response.body)
  @original_status = @body.map { |item| item['original_status'].to_i }
  @status_code = response.code.to_i
  @cb_status = @body.map { |item| StatusResolution.resolve_cb_status(item) }
  @pc_status = @cb_status
  @url = @body.map { |item| item['url'] }
  @rid = @body.map { |item| item['rid'] }
  @stored_at = @body.map { |item| item['stored_at'] }

  self
end

#delete(rid) ⇒ Object

Raises:



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/crawlbase/storage_api.rb', line 54

def delete(rid)
  raise INVALID_RID if rid.nil? || rid.empty?

  uri = URI(BASE_URL)
  uri.query = URI.encode_www_form(token: token, rid: rid)
  http = Net::HTTP.new(uri.host)
  request = Net::HTTP::Delete.new(uri.request_uri)
  response = http.request(request)

  @url, @original_status, @cb_status, @pc_status, @stored_at = nil
  @status_code = response.code.to_i
  @rid = rid
  @body = JSON.parse(response.body)

  @body.key?('success')
end

#get(url_or_rid, format = 'html') ⇒ Object

Raises:



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/crawlbase/storage_api.rb', line 32

def get(url_or_rid, format = 'html')
  raise INVALID_URL_OR_RID if url_or_rid.nil? || url_or_rid.empty?

  uri = URI(BASE_URL)
  uri.query = URI.encode_www_form({ token: token, format: format }.merge(decide_url_or_rid(url_or_rid)))
  response = Net::HTTP.get_response(uri)

  res = format == 'json' ? JSON.parse(response.body) : response

  @original_status = res['original_status'].to_i
  @cb_status = StatusResolution.resolve_cb_status(res)
  @pc_status = @cb_status
  @url = res['url']
  @rid = res['rid']
  @stored_at = res['stored_at']

  @status_code = response.code.to_i
  @body = response.body

  self
end

#pc_status ⇒ Object

Deprecated: use #cb_status. Returns the same resolved Crawlbase status.



27
28
29
30
 
# File 'lib/crawlbase/storage_api.rb', line 27

def pc_status
  StatusResolution.warn_pc_status_deprecated
  @pc_status
end

#rids(limit = -1)) ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/crawlbase/storage_api.rb', line 93

def rids(limit = -1)
  uri = URI("#{BASE_URL}/rids")
  query_hash = { token: token }
  query_hash.merge!({ limit: limit }) if limit >= 0
  uri.query = URI.encode_www_form(query_hash)

  response = Net::HTTP.get_response(uri)
  @url, @original_status, @cb_status, @pc_status, @stored_at = nil
  @status_code = response.code.to_i
  @body = JSON.parse(response.body)
  @rid = @body

  @body
end

#total_count ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/crawlbase/storage_api.rb', line 108

def total_count
  uri = URI("#{BASE_URL}/total_count")
  uri.query = URI.encode_www_form(token: token)

  response = Net::HTTP.get_response(uri)
  @url, @original_status, @cb_status, @pc_status, @stored_at = nil
  @status_code = response.code.to_i
  @rid = rid
  @body = JSON.parse(response.body)

  body['totalCount']
end