Module: Spidy::Connector

Extended by:
ActiveSupport::Autoload
Defined in:
lib/spidy/connector.rb

Overview

This class is responsible for actually making a network connection and downloading hypertext

Defined Under Namespace

Modules: StaticAccessor Classes: Direct, Html, Json, Lightpanda, Retry, RetryableCaller, TorConnector, Xml

Constant Summary collapse

DEFAULT_WAIT_TIME = 
5
USER_AGENT =

default user agent

[
  'Mozilla/5.0',
  '(Macintosh; Intel Mac OS X 10_12_6)',
  'AppleWebKit/537.36',
  '(KHTML, like Gecko)',
  'Chrome/64.0.3282.186',
  'Safari/537.36'
].join(' ')
DEFAULT_LOGGER =

error output logger

proc { |values| warn(values.to_json) }

Class Method Summary collapse

Class Method Details

.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil) ⇒ Object 



125
126
127
128
129
130
131
132
133
134
 
# File 'lib/spidy/connector.rb', line 125

def self.get(value, wait_time: nil, user_agent: nil, socks_proxy: nil, logger: nil)
  user_agent ||= USER_AGENT
  logger ||= DEFAULT_LOGGER
  wait_time ||= DEFAULT_WAIT_TIME

  connector = get_connector(value, user_agent: user_agent, socks_proxy: socks_proxy)
  return connector if connector.is_a?(Spidy::Connector::Direct)

  RetryableCaller.new(connector, wait_time: wait_time, logger: logger)
end

.get_connector(value, user_agent: nil, socks_proxy: nil) ⇒ Object

get connection handller



139
140
141
142
143
144
145
146
147
 
# File 'lib/spidy/connector.rb', line 139

def self.get_connector(value, user_agent: nil, socks_proxy: nil)
  return value if value.respond_to?(:call)

  connector = const_get(value.to_s.classify).new(user_agent: user_agent)
  fail "Not defined connnector[#{value}]" if connector.nil?
  return connector if socks_proxy.nil?

  TorConnector.new(connector, socks_proxy)
end