Class: SourceMonitor::Scraping::Scheduler

Inherits:
Object
  • Object
show all
Defined in:
lib/source_monitor/scraping/scheduler.rb

Overview

Identifies items that still need scraping and enqueue jobs for sources configured for automatic scraping. This mirrors the feed fetch scheduler so recurring tasks can keep the scrape queue warm.

Constant Summary collapse

DEFAULT_BATCH_SIZE = 
100

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(limit:) ⇒ Scheduler

Returns a new instance of Scheduler.



15
16
17
 
# File 'lib/source_monitor/scraping/scheduler.rb', line 15

def initialize(limit:)
  @limit = limit
end

Class Method Details

.run(limit: DEFAULT_BATCH_SIZE) ⇒ Object 



11
12
13
 
# File 'lib/source_monitor/scraping/scheduler.rb', line 11

def self.run(limit: DEFAULT_BATCH_SIZE)
  new(limit:).run
end

Instance Method Details

#run ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/source_monitor/scraping/scheduler.rb', line 19

def run
  items = due_items.limit(limit).includes(:source).to_a
  return 0 if items.empty?

  items.sum do |item|
    result = SourceMonitor::Scraping::Enqueuer.enqueue(item: item, source: item.source, reason: :auto)
    result.enqueued? ? 1 : 0
  end
rescue StandardError => error
  Rails.logger.warn(
    "[SourceMonitor::Scraping::Scheduler] Scheduler run failed: #{error.class} - #{error.message}"
  ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
  0
end