Class: Ductwork::Process

Inherits:
Record
  • Object
show all
Defined in:
lib/ductwork/models/process.rb

Constant Summary collapse

ORPHANED_CLAIM_SWEEP_MULTIPLIER = 
3

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.adopt_or_create_current!(role) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/ductwork/models/process.rb', line 27

def self.adopt_or_create_current!(role)
  pid = ::Process.pid
  machine_identifier = Ductwork::MachineIdentifier.fetch
  last_heartbeat_at = Ductwork::DatabaseClock.now
  existing = Ductwork::Process.find_by(pid:, machine_identifier:)

  # NOTE: Same pid + machine_identifier can only mean the OS reused this
  # pid, which happens after the prior process at this identity has
  # exited -- a live process can never be replaced while still running.
  # So its in-flight claims belong to a dead incarnation and must always
  # be recovered here, even when the heartbeat still looks fresh because
  # the reaper's timeout hasn't elapsed yet.
  existing&.recover_crashed_claims!(role)

  Ductwork::Process
    .find_or_initialize_by(pid:, machine_identifier:)
    .tap { |process| process.update!(last_heartbeat_at:, role:) }
end

.current ⇒ Object 



46
47
48
49
50
51
 
# File 'lib/ductwork/models/process.rb', line 46

def self.current
  pid = ::Process.pid
  machine_identifier = Ductwork::MachineIdentifier.fetch

  find_by(pid:, machine_identifier:)
end

.reap_all!(role) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/ductwork/models/process.rb', line 53

def self.reap_all!(role)
  count = 0
  timeout = Ductwork.configuration.supervisor_reaper_timeout
  sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", timeout)

  Ductwork.logger.debug(
    msg: "Reaping orphaned process records",
    role: role
  )

  where(sql).find_each do |process|
    process.reap!(role)
    count += 1
  end

  Ductwork.logger.debug(
    msg: "Reaped #{count} orphaned process records",
    count: count,
    role: role
  )
end

.report_heartbeat!(role) ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/ductwork/models/process.rb', line 119

def self.report_heartbeat!(role)
  process = current

  if process.present?
    process.update!(last_heartbeat_at: Ductwork::DatabaseClock.now)
    process
  else
    Ductwork.logger.warn(
      msg: "Process record missing, re-adopting (likely reaped after host suspend)",
      pid: ::Process.pid
    )
    adopt_or_create_current!(role)
  end
end

.sweep_orphaned_claims!(role) ⇒ Object

NOTE: backstop for claims that lost their process_id (dependent: :nullify on a process destroy racing a fresh claim) and so are unreachable through any Process association -- reap!/reap_all! above only ever look through a Process record's own advancements/executions, and a nil process_id means there is no Process record to look through. This scans the orphaned rows directly instead.

Executions are additionally scoped to availabilities with a completed_at present: an unclaimed execution legitimately has process_id: nil while it waits to be picked up (see RowLockingExecutionClaim#claim_availability, which only sets process_id at claim time), so process_id: nil alone would misidentify perfectly healthy queued work as orphaned.



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/ductwork/models/process.rb', line 87

def self.sweep_orphaned_claims!(role)
  timeout = Ductwork.configuration.supervisor_reaper_timeout * ORPHANED_CLAIM_SWEEP_MULTIPLIER
  count = 0
  advancement_sql = Ductwork::DatabaseClock.ago_sql("started_at", timeout)

  Ductwork::Advancement
    .where(process_id: nil, completed_at: nil)
    .where(advancement_sql)
    .find_each do |advancement|
      advancement.process_crashed!
      count += 1
    end

  execution_sql = Ductwork::DatabaseClock.ago_sql("ductwork_availabilities.completed_at", timeout)

  Ductwork::Execution
    .joins(:availability)
    .where(process_id: nil, completed_at: nil)
    .where.not(ductwork_availabilities: { completed_at: nil })
    .where(execution_sql)
    .find_each do |execution|
      execution.crashed!
      count += 1
    end

  Ductwork.logger.debug(
    msg: "Swept #{count} orphaned claims",
    count: count,
    role: role
  )
end

Instance Method Details

#healthy? ⇒ Boolean

Returns:

  • (Boolean) 


184
185
186
187
188
189
 
# File 'lib/ductwork/models/process.rb', line 184

def healthy?
  timeout = Ductwork.configuration.supervisor_reaper_timeout
  sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", timeout)

  self.class.where(id:).where(sql).none?
end

#reap!(role, force: false) ⇒ Object 



134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/ductwork/models/process.rb', line 134

def reap!(role, force: false)
  timeout = Ductwork.configuration.supervisor_reaper_timeout
  sql = Ductwork::DatabaseClock.ago_sql("last_heartbeat_at", timeout)

  Ductwork.logger.debug(
    msg: "Reaping orphaned process record #{id}",
    id: id,
    role: role
  )

  Ductwork::Record.transaction do
    # NOTE: Callers that have already killed/stopped the process hold proof
    # of death and pass force: true to skip the staleness guard. The row
    # lock and existence check are kept either way to stay atomic and to
    # avoid double-reaping a record another parent already cleaned up
    scope = Ductwork::Process.where(id:).lock
    scope = scope.where(sql) unless force

    return unless scope.exists?

    recover_crashed_claims!(role)
    destroy
  end

  Ductwork.logger.debug(
    msg: "Reaped orphaned process record #{id}",
    id: id,
    role: role
  )
rescue ActiveRecord::RecordNotFound
  Ductwork.logger.debug(
    msg: "Process already reaped by another parent",
    id: id,
    role: role
  )
end

#recover_crashed_claims!(role) ⇒ Object 



171
172
173
174
175
176
177
178
179
180
181
182
 
# File 'lib/ductwork/models/process.rb', line 171

def recover_crashed_claims!(role)
  Ductwork.logger.debug(
    msg: "Recovering in-flight claims on reused process record #{id}",
    id: id,
    role: role
  )

  Ductwork::Record.transaction do
    advancements.where(completed_at: nil).find_each(&:process_crashed!)
    executions.where(completed_at: nil).find_each(&:crashed!)
  end
end