Class: Vessel::Driver

Inherits:
Object
  • Object
show all
Defined in:
lib/vessel/driver.rb,
lib/vessel/driver/page.rb,
lib/vessel/driver/registry.rb,
lib/vessel/driver/ferrum/page.rb,
lib/vessel/driver/ferrum/driver.rb,
lib/vessel/driver/mechanize/page.rb,
lib/vessel/driver/mechanize/driver.rb

Direct Known Subclasses

Ferrum::Driver, Mechanize::Driver

Defined Under Namespace

Modules: Ferrum, Mechanize Classes: Page, Registry

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(settings) ⇒ Driver

Returns a new instance of Driver.



29
30
31
32
33
34
35
36
 
# File 'lib/vessel/driver.rb', line 29

def initialize(settings)
  @last_request = nil
  @settings = settings
  @visited_urls = Concurrent::Map.new { 0 }
  @proxy = settings[:proxy]&.new
  start
  at_exit { stop }
end

Instance Attribute Details

#browser ⇒ Object (readonly)

Returns the value of attribute browser.



27
28
29
 
# File 'lib/vessel/driver.rb', line 27

def browser
  @browser
end

#proxy ⇒ Object (readonly)

Returns the value of attribute proxy.



27
28
29
 
# File 'lib/vessel/driver.rb', line 27

def proxy
  @proxy
end

#settings ⇒ Object (readonly)

Returns the value of attribute settings.



27
28
29
 
# File 'lib/vessel/driver.rb', line 27

def settings
  @settings
end

Class Method Details

.direct_network_errors ⇒ Object 



12
13
14
15
16
17
18
19
20
21
 
# File 'lib/vessel/driver.rb', line 12

def self.direct_network_errors
  [
    ::Net::HTTP::Persistent::Error,
    ::Net::HTTPFatalError,
    ::Net::ReadTimeout,
    ::Net::OpenTimeout,
    ::SocketError,
    ::Errno::ECONNRESET
  ].freeze
end

.driver_name(name) ⇒ Object 



8
9
10
 
# File 'lib/vessel/driver.rb', line 8

def self.driver_name(name)
  Registry.instance.register(name, self)
end

.indirect_network_errors ⇒ Object 



23
24
25
 
# File 'lib/vessel/driver.rb', line 23

def self.indirect_network_errors
  [].freeze
end

Instance Method Details

#create_page(proxy: nil) ⇒ Object

Raises:



46
47
48
 
# File 'lib/vessel/driver.rb', line 46

def create_page(proxy: nil)
  raise NotImplementedError
end

#go_to(request) ⇒ Object 



55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/vessel/driver.rb', line 55

def go_to(request)
  url = request.url.to_s
  response = Response.new(**request.to_handler)
  return [response, request] if request.stub?

  # The url is registered before the page is created, otherwise threads
  # racing for the same url all pass the check and visit it in parallel.
  attempt = visit(url)
  if request.once && attempt > 1
    Logger.info("Driver: rejecting #{url} visited #{attempt - 1} time(s)")
    response.rejected = true
    return [response, request]
  end

  begin
    page = prepare_page(request)
    delay(request.delay)
    Logger.info("Driver: visiting #{url}, last_request = #{@last_request.to_i}")
    page.go_to(url)
    @last_request = Time.now.to_i
    response = Response.new(page: page, attempt: attempt, **request.to_handler)
    [response, request]
  rescue StandardError => e
    if network_error?(e)
      Logger.error("Driver: network issue for #{url}, attempt ##{attempt}")
      Logger.error("Driver: #{e.class}: #{e.message}")
      if attempt < settings[:network_error_attempts]
        attempt = visit(url)
        restart
        retry
      end
    end

    [response, request, e]
  end
end

#restart ⇒ Object 



50
51
52
53
 
# File 'lib/vessel/driver.rb', line 50

def restart
  stop
  start
end

#start ⇒ Object

Raises:



38
39
40
 
# File 'lib/vessel/driver.rb', line 38

def start
  raise NotImplementedError
end

#stop ⇒ Object

Raises:



42
43
44
 
# File 'lib/vessel/driver.rb', line 42

def stop
  raise NotImplementedError
end