Module: Crawlscope::FetchExecutor

Defined in:
lib/crawlscope/fetch_executor.rb,
lib/crawlscope/fetch_executor/async.rb,
lib/crawlscope/fetch_executor/threaded.rb

Defined Under Namespace

Classes: Async, Threaded

Constant Summary collapse

NAMES = 
%i[threaded async].freeze

Class Method Summary collapse

Class Method Details

.build(name:, concurrency:) ⇒ Object 



9
10
11
12
13
14
15
16
17
18
 
# File 'lib/crawlscope/fetch_executor.rb', line 9

def build(name:, concurrency:)
  return name if name.respond_to?(:call)

  case normalized_name(name)
  when :threaded
    Threaded.new(concurrency: concurrency)
  when :async
    Async.new(concurrency: concurrency)
  end
end

.map(name:, concurrency:, items:, &block) ⇒ Object 



20
21
22
23
24
25
 
# File 'lib/crawlscope/fetch_executor.rb', line 20

def map(name:, concurrency:, items:, &block)
  items = Array(items)
  return items.map(&block) if items.size < 2 || concurrency.to_i <= 1

  build(name: name, concurrency: concurrency).call(items, &block)
end

.normalize(name) ⇒ Object 



27
28
29
30
31
 
# File 'lib/crawlscope/fetch_executor.rb', line 27

def normalize(name)
  return name if name.respond_to?(:call)

  normalized_name(name)
end

.normalized_name(name) ⇒ Object

Raises:



33
34
35
36
37
38
39
40
41
 
# File 'lib/crawlscope/fetch_executor.rb', line 33

def normalized_name(name)
  normalized = name.to_s.strip
  normalized = "threaded" if normalized.empty?

  value = normalized.to_sym
  return value if NAMES.include?(value)

  raise ConfigurationError, "Crawlscope fetch_executor must be threaded or async"
end