Class: Crawlscope::Crawler

Inherits:
Object
  • Object
show all
Defined in:
lib/crawlscope/crawler.rb

Instance Method Summary collapse

Constructor Details

#initialize(page_fetcher:, concurrency:) ⇒ Crawler

Returns a new instance of Crawler.



7
8
9
10
 
# File 'lib/crawlscope/crawler.rb', line 7

def initialize(page_fetcher:, concurrency:)
  @page_fetcher = page_fetcher
  @concurrency = concurrency
end

Instance Method Details

#call(urls) ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'lib/crawlscope/crawler.rb', line 12

def call(urls)
  pages = Concurrent::Array.new
  pool = Concurrent::FixedThreadPool.new(@concurrency)

  urls.each do |url|
    pool.post do
      pages << @page_fetcher.fetch(url)
    end
  end

  pool.shutdown
  pool.wait_for_termination

  pages.to_a
end