Class: Vessel::Cargo

Inherits:
Object
  • Object
show all
Extended by:
Forwardable, Settings
Includes:
Callbacks, Fieldable
Defined in:
lib/vessel/cargo.rb,
lib/vessel/cargo/field.rb,
lib/vessel/cargo/fields.rb,
lib/vessel/cargo/settings.rb,
lib/vessel/cargo/callbacks.rb,
lib/vessel/cargo/fieldable.rb,
lib/vessel/cargo/field_type.rb

Direct Known Subclasses

ApplicationCrawler

Defined Under Namespace

Modules: Callbacks, Fieldable, Settings Classes: Field, FieldType, Fields

Constant Summary

Constants included from Settings

Settings::DELAY, Settings::MAX_THREADS, Settings::MIDDLEWARE, Settings::MIN_THREADS, Settings::NETWORK_ERROR_ATTEMPTS, Settings::START_URLS

Constants included from Callbacks

Callbacks::INFO_INTERVAL

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Settings

allow_cookies, blacklist, cookie, cookies, delay, driver, headers, middleware, name, network_error_attempts, proxy, start_urls, threads, to_domain_name, whitelist

Methods included from Util

#deep_clone

Methods included from Fieldable

#field, #fields

Methods included from Callbacks

#after, #after_change, #before, #before_start, #before_stop, #info, #on_error

Constructor Details

#initialize(response = nil) ⇒ Cargo

Returns a new instance of Cargo.



44
45
46
47
48
49
50
 
# File 'lib/vessel/cargo.rb', line 44

def initialize(response = nil)
  @settings = self.class.settings
  @domain = settings[:domain]
  @response = response
  @page = response&.page
  @size = response&.size
end

Instance Attribute Details

#domain ⇒ Object (readonly)

Returns the value of attribute domain.



42
43
44
 
# File 'lib/vessel/cargo.rb', line 42

def domain
  @domain
end

#page ⇒ Object (readonly)

Returns the value of attribute page.



42
43
44
 
# File 'lib/vessel/cargo.rb', line 42

def page
  @page
end

#response ⇒ Object (readonly)

Returns the value of attribute response.



42
43
44
 
# File 'lib/vessel/cargo.rb', line 42

def response
  @response
end

#settings ⇒ Object (readonly)

Returns the value of attribute settings.



42
43
44
 
# File 'lib/vessel/cargo.rb', line 42

def settings
  @settings
end

#size ⇒ Object (readonly)

Returns the value of attribute size.



42
43
44
 
# File 'lib/vessel/cargo.rb', line 42

def size
  @size
end

Class Method Details

.build_request(url:, **options) ⇒ Object 



15
16
17
18
19
20
21
 
# File 'lib/vessel/cargo.rb', line 15

def self.build_request(url:, **options)
  Request.new(url: url,
              delay: settings[:delay],
              cookies: settings[:cookies],
              headers: settings[:headers],
              **options)
end

.run(settings = nil) ⇒ Object 



10
11
12
13
 
# File 'lib/vessel/cargo.rb', line 10

def self.run(settings = nil, &)
  self.settings.merge!(Hash(settings))
  Engine.run(self, &)
end

.start_requests ⇒ Object 



23
24
25
26
27
 
# File 'lib/vessel/cargo.rb', line 23

def self.start_requests
  return build_request(url: nil) if settings[:start_urls].empty?

  settings[:start_urls].map { |u, h| build_request(url: u, handler: h) }
end

.store_cookies(cookies) ⇒ Object 



29
30
31
 
# File 'lib/vessel/cargo.rb', line 29

def self.store_cookies(cookies)
  settings[:cookies] = cookies
end

Instance Method Details

#parse ⇒ Object

Raises:



52
53
54
 
# File 'lib/vessel/cargo.rb', line 52

def parse
  raise NotImplementedError
end