Class: Vessel::Engine

Inherits:
Object
  • Object
show all
Includes:
Stats
Defined in:
lib/vessel/engine.rb,
lib/vessel/engine/stats.rb

Defined Under Namespace

Modules: Stats

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(klass) ⇒ Engine

Returns a new instance of Engine.



15
16
17
18
19
20
21
 
# File 'lib/vessel/engine.rb', line 15

def initialize(klass, &)
  @crawler_class = klass
  @settings = klass.settings
  @queue = SizedQueue.new(settings[:max_threads])
  @scheduler = Scheduler.new(@queue, settings)
  @middleware_scheduler = MiddlewareScheduler.new(settings, &)
end

Instance Attribute Details

#crawler_class ⇒ Object (readonly)

Returns the value of attribute crawler_class.



13
14
15
 
# File 'lib/vessel/engine.rb', line 13

def crawler_class
  @crawler_class
end

#middleware_scheduler ⇒ Object (readonly)

Returns the value of attribute middleware_scheduler.



13
14
15
 
# File 'lib/vessel/engine.rb', line 13

def middleware_scheduler
  @middleware_scheduler
end

#scheduler ⇒ Object (readonly)

Returns the value of attribute scheduler.



13
14
15
 
# File 'lib/vessel/engine.rb', line 13

def scheduler
  @scheduler
end

#settings ⇒ Object (readonly)

Returns the value of attribute settings.



13
14
15
 
# File 'lib/vessel/engine.rb', line 13

def settings
  @settings
end

Class Method Details

.run ⇒ Object 



9
10
11
 
# File 'lib/vessel/engine.rb', line 9

def self.run(...)
  new(...).tap(&:run)
end

Instance Method Details

#handle(response, request, error) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/vessel/engine.rb', line 54

def handle(response, request, error)
  increase(:res_dequeued)
  return if response.rejected

  crawler = crawler_class.new(response)
  crawler_class.store_cookies(response.cookies) if settings[:allow_cookies]
  args = error ? [:on_error, request, error] : [request.handler].compact
  Logger.debug("Engine: crawler starts processing #{response.url} with :#{response.handler}")
  crawler.send(*args) do |*result|
    result = result.flatten
    if result.all?(Request)
      Logger.debug("Engine: :#{response.handler} enqueued requests #{result.map { |r| r.url.to_s }}")
      schedule(result)
    else
      result.each { |fields| process(fields) }
    end
  end
  Logger.debug("Engine: crawler stopped processing #{response.url}")
rescue StandardError => e
  Logger.error("Engine: there was an issue while processing #{response.url}")
  Logger.error("Engine: `#{e.class}: #{e.message}`\n#{e.backtrace.join("\n")}")
ensure
  increase(:res_handled)
  response.close
end

#parse(url, handler, data = nil) ⇒ Object 



44
45
46
47
48
49
50
51
52
 
# File 'lib/vessel/engine.rb', line 44

def parse(url, handler, data = nil)
  trap_sigint
  request = crawler_class.build_request(url: url, handler: handler, data: data)
  schedule(request)
  event_loop
ensure
  scheduler.stop
  middleware_scheduler.stop
end

#run ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/vessel/engine.rb', line 23

def run
  at_exit do
    crawler_class.new.info(stats)
    crawler_class.new.before_stop
  end

  trap_sigint
  crawler_class.new.before_start
  crawler_class.new.before(stats)
  @info_task = Concurrent::TimerTask.execute(execution_interval: crawler_class::INFO_INTERVAL) do
    crawler_class.new.info(stats)
  end
  schedule(crawler_class.start_requests)
  event_loop
ensure
  @info_task&.shutdown
  scheduler.stop
  middleware_scheduler.stop
  crawler_class.new.after(stats)
end