Module: Vessel::Cargo::Settings

Includes:
Util
Included in:
Vessel::Cargo
Defined in:
lib/vessel/cargo/settings.rb

Constant Summary collapse

DELAY = 
0
START_URLS = 
{}.freeze
MIDDLEWARE = 
[].freeze
MIN_THREADS = 
1
MAX_THREADS = 
Concurrent.processor_count
NETWORK_ERROR_ATTEMPTS = 
5

Instance Method Summary collapse

Methods included from Util

#deep_clone

Instance Method Details

#allow_cookies(value) ⇒ Object 



83
84
85
 
# File 'lib/vessel/cargo/settings.rb', line 83

def allow_cookies(value)
  settings[:allow_cookies] = value
end

#blacklist(patterns) ⇒ Object 



71
72
73
 
# File 'lib/vessel/cargo/settings.rb', line 71

def blacklist(patterns)
  settings[:blacklist] = patterns
end

#cookie(name:, value:, domain: nil, path: nil, httponly: nil, secure: nil, expires: nil) ⇒ Object 



46
47
48
49
50
51
52
 
# File 'lib/vessel/cargo/settings.rb', line 46

def cookie(name:, value:, domain: nil, path: nil, httponly: nil, secure: nil, expires: nil)
  settings[:cookies] ||= []
  domain ||= settings[:domain]
  cookie = { name: name, value: value, domain: domain, path: path,
             httponly: httponly, secure: secure, expires: expires }.compact
  settings[:cookies] << cookie
end

#cookies(*value) ⇒ Object 



54
55
56
 
# File 'lib/vessel/cargo/settings.rb', line 54

def cookies(*value)
  value.flatten.each { |c| cookie(**c) }
end

#delay(value) ⇒ Object 



38
39
40
 
# File 'lib/vessel/cargo/settings.rb', line 38

def delay(value)
  settings[:delay] = value
end

#domain(value) ⇒ Object 



22
23
24
25
 
# File 'lib/vessel/cargo/settings.rb', line 22

def domain(value)
  settings[:domain] = value
  Vessel.loader.register(name, self)
end

#driver(name, **options) ⇒ Object 



33
34
35
36
 
# File 'lib/vessel/cargo/settings.rb', line 33

def driver(name, **options)
  settings[:driver_name] = name.to_sym
  settings[:driver_options] = options
end

#headers(value) ⇒ Object 



42
43
44
 
# File 'lib/vessel/cargo/settings.rb', line 42

def headers(value)
  settings[:headers] = value
end

#middleware(*classes) ⇒ Object 



63
64
65
 
# File 'lib/vessel/cargo/settings.rb', line 63

def middleware(*classes)
  settings[:middleware] = classes
end

#name ⇒ Object 



18
19
20
 
# File 'lib/vessel/cargo/settings.rb', line 18

def name
  settings[:domain]
end

#network_error_attempts(value) ⇒ Object 



79
80
81
 
# File 'lib/vessel/cargo/settings.rb', line 79

def network_error_attempts(value)
  settings[:network_error_attempts] = value.to_i
end

#proxy(klass) ⇒ Object 



67
68
69
 
# File 'lib/vessel/cargo/settings.rb', line 67

def proxy(klass)
  settings[:proxy] = klass
end

#settings ⇒ Object 



87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/vessel/cargo/settings.rb', line 87

def settings
  @settings ||= if superclass.respond_to?(:settings)
                  deep_clone(superclass.settings)
                else
                  {
                    delay: DELAY, start_urls: START_URLS, middleware: MIDDLEWARE,
                    min_threads: MIN_THREADS, max_threads: MAX_THREADS,
                    driver_name: :ferrum, driver_options: {}, headers: nil,
                    cookies: nil, allow_cookies: true, proxy: nil,
                    blacklist: nil, whitelist: nil, domain: nil,
                    network_error_attempts: NETWORK_ERROR_ATTEMPTS
                  }
                end
end

#start_urls(*url_handlers) ⇒ Object 



27
28
29
30
31
 
# File 'lib/vessel/cargo/settings.rb', line 27

def start_urls(*url_handlers)
  settings[:start_urls] = url_handlers.compact.flatten.inject({}) do |options, url_handler|
    options.merge(url_handler.is_a?(Hash) ? url_handler : { url_handler => Request::DEFAULT_HANDLER })
  end
end

#threads(min: MIN_THREADS, max: MAX_THREADS) ⇒ Object 



58
59
60
61
 
# File 'lib/vessel/cargo/settings.rb', line 58

def threads(min: MIN_THREADS, max: MAX_THREADS)
  settings[:min_threads] = min
  settings[:max_threads] = max
end

#to_domain_name(klass) ⇒ Object 



102
103
104
105
106
107
108
109
110
 
# File 'lib/vessel/cargo/settings.rb', line 102

def to_domain_name(klass)
  # name&.split("::")&.last&.downcase
  klass.name.gsub(/[a-zA-Z](?=[A-Z])/, '\0 ').downcase.split.map do |value|
    next "." if value == "dot"
    next "-" if value == "dash"

    value
  end.join
end

#whitelist(patterns) ⇒ Object 



75
76
77
 
# File 'lib/vessel/cargo/settings.rb', line 75

def whitelist(patterns)
  settings[:whitelist] = patterns
end