Class: Yes::Core::Jobs::ReadModelRecoveryJob

Inherits:
ActiveJob::Base
  • Object
show all
Defined in:
lib/yes/core/jobs/read_model_recovery_job.rb

Overview

Background job that runs periodically to recover stuck read models This job should be scheduled to run every 30 seconds via cron or similar

Constant Summary collapse

MAX_CONSECUTIVE_FAILURES =

Circuit breaker configuration

5
CIRCUIT_BREAKER_TIMEOUT = 
5.minutes

Class Attribute Summary collapse

Instance Method Summary collapse

Class Attribute Details

.circuit_opened_at ⇒ Object

Track circuit breaker state in memory (or Redis in production)



17
18
19
 
# File 'lib/yes/core/jobs/read_model_recovery_job.rb', line 17

def circuit_opened_at
  @circuit_opened_at
end

.consecutive_failures ⇒ Object

Track circuit breaker state in memory (or Redis in production)



17
18
19
 
# File 'lib/yes/core/jobs/read_model_recovery_job.rb', line 17

def consecutive_failures
  @consecutive_failures
end

Instance Method Details

#perform(stuck_timeout_minutes: 1, batch_size: 100) ⇒ Object

Performs the recovery of stuck read models

Parameters:

  • stuck_timeout (Integer) —

    Minutes after which a model is considered stuck (default: 1)

  • batch_size (Integer) (defaults to: 100) —

    Number of read models to process at once (default: 100)



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/yes/core/jobs/read_model_recovery_job.rb', line 26

def perform(stuck_timeout_minutes: 1, batch_size: 100)
  # Check circuit breaker
  if circuit_open?
    Rails.logger.warn('ReadModelRecoveryJob circuit breaker is open, skipping execution')
    return
  end

  stuck_timeout = stuck_timeout_minutes.minutes

  Rails.logger.info("Starting read model recovery scan (timeout: #{stuck_timeout_minutes} minutes)")

  results = Yes::Core::CommandHandling::ReadModelRecoveryService.recover_all_stuck_read_models(
    stuck_timeout:,
    batch_size:
  )

  process_results(results)

  # Reset circuit breaker on success
  self.class.consecutive_failures = 0

  Rails.logger.info("Read model recovery scan completed: #{results.size} models processed")
rescue ActiveRecord::ActiveRecordError => e
  # Database-related errors (connection issues, deadlocks, etc.)
  Rails.logger.error("Database error during read model recovery: #{e.message}")
  handle_job_failure(e)
  raise
rescue PgEventstore::Error => e
  # Event store related errors (revision conflicts, stream errors)
  Rails.logger.error("Event store error during read model recovery: #{e.message}")
  handle_job_failure(e)
  raise
rescue NameError, NoMethodError => e
  # Configuration or class loading errors - these should not trigger circuit breaker
  Rails.logger.error("Configuration error during read model recovery: #{e.message}")
  Rails.logger.error('This likely indicates a misconfiguration - please check your read model classes')
  # Don't increment circuit breaker for configuration errors
  raise
rescue StandardError => e
  # Unexpected errors - log with full backtrace for debugging
  Rails.logger.error("Unexpected error during read model recovery: #{e.class.name} - #{e.message}")
  Rails.logger.error(e.backtrace.join("\n"))

  # Still handle as a failure for circuit breaker
  handle_job_failure(e)

  # Re-raise to ensure job framework knows it failed
  raise
end