Class: Tina4::QueueBackends::MongoBackend

Inherits:
Object
  • Object
show all
Defined in:
lib/tina4/queue_backends/mongo_backend.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ MongoBackend

Returns a new instance of MongoBackend.



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 10

def initialize(options = {})
  require "mongo"
  @max_retries = options[:max_retries] || 3
  # Seconds to delay a requeued (failed/retried) job before it is eligible
  # again. Default 0 = available on the very next dequeue, so a fail()'d job
  # retries immediately (matching the lite backend) instead of waiting out
  # the visibility window.
  @retry_backoff = (options[:retry_backoff] || 0).to_f

  uri = options[:uri] || ENV["TINA4_MONGO_URI"]
  host = options[:host] || ENV.fetch("TINA4_MONGO_HOST", "localhost")
  port = (options[:port] || ENV.fetch("TINA4_MONGO_PORT", 27017)).to_i
  username = options[:username] || ENV["TINA4_MONGO_USERNAME"]
  password = options[:password] || ENV["TINA4_MONGO_PASSWORD"]
  db_name = options[:db] || ENV.fetch("TINA4_MONGO_DB", "tina4")
  @collection_name = options[:collection] || ENV.fetch("TINA4_MONGO_COLLECTION", "tina4_queue")
  # Reservation/visibility timeout (seconds): a dequeued message is held
  # reserved (status "processing") with available_at = now + timeout;
  # reclaim_expired returns it once that passes (consumer died mid-flight).
  # <= 0 disables reclaim.
  @visibility_timeout = resolve_visibility_timeout(options[:visibility_timeout])

  if uri
    @client = Mongo::Client.new(uri)
  else
    conn_options = { database: db_name }
    conn_options[:user] = username if username
    conn_options[:password] = password if password
    @client = Mongo::Client.new(["#{host}:#{port}"], conn_options)
  end

  @db = @client.database
  create_indexes
rescue LoadError
  raise "MongoDB backend requires the 'mongo' gem. Install with: gem install mongo"
end

Instance Attribute Details

#max_retries ⇒ Object

Reservation/visibility + retry policy (settable so a Queue can propagate its own onto a backend instance passed directly — legacy path).



8
9
10
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 8

def max_retries
  @max_retries
end

#retry_backoff ⇒ Object

Reservation/visibility + retry policy (settable so a Queue can propagate its own onto a backend instance passed directly — legacy path).



8
9
10
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 8

def retry_backoff
  @retry_backoff
end

#visibility_timeout ⇒ Object

Reservation/visibility + retry policy (settable so a Queue can propagate its own onto a backend instance passed directly — legacy path).



8
9
10
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 8

def visibility_timeout
  @visibility_timeout
end

Instance Method Details

#acknowledge(message) ⇒ Object 



138
139
140
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 138

def acknowledge(message)
  collection.delete_one(_id: message.id)
end

#close ⇒ Object 



237
238
239
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 237

def close
  @client&.close
end

#complete(message) ⇒ Object

Terminal success — the job is done and removed (mirrors the lite backend’s complete()). Without this, job.complete() was a no-op on MongoDB and the document stayed “processing” forever.



145
146
147
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 145

def complete(message)
  collection.delete_one(_id: message.id)
end

#dead_letter(message) ⇒ Object 



198
199
200
201
202
203
204
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 198

def dead_letter(message)
  collection.find_one_and_update(
    { _id: message.id },
    { "$set" => { status: "dead", topic: "#{message.topic}.dead_letter" } },
    upsert: true
  )
end

#dead_letters(topic, max_retries: 3) ⇒ Object 



210
211
212
213
214
215
216
217
218
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 210

def dead_letters(topic, max_retries: 3)
  collection.find(topic: "#{topic}.dead_letter", status: "dead").map do |doc|
    Tina4::Job.new(
      topic: doc["topic"],
      payload: doc["payload"],
      id: doc["_id"]
    )
  end
end

#dequeue(topic) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 66

def dequeue(topic)
  # Reclaim any reservations whose consumer died before acking, then take
  # the next available message (at-least-once delivery).
  reclaim_expired(topic, @max_retries)
  now = Time.now.utc
  # The claim advances available_at to now + visibility_timeout and records
  # reserved_at so reclaim_expired can return the job if the consumer dies
  # before acknowledge/complete. This is the fix for the "reserved forever"
  # bug — previously available_at was left unchanged.
  doc = collection.find_one_and_update(
    { topic: topic, status: "pending" },
    { "$set" => {
      status: "processing",
      reserved_at: now,
      available_at: now + (@visibility_timeout || 0)
    } },
    sort: { created_at: 1 },
    return_document: :after
  )
  return nil unless doc

  Tina4::Job.new(
    topic: doc["topic"],
    payload: doc["payload"],
    id: doc["_id"],
    priority: doc["priority"] || 0,
    attempts: doc["attempts"] || 0
  )
end

#enqueue(message) ⇒ Object 



55
56
57
58
59
60
61
62
63
64
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 55

def enqueue(message)
  collection.insert_one(
    _id: message.id,
    topic: message.topic,
    payload: message.payload,
    created_at: message.created_at.utc,
    attempts: message.attempts,
    status: "pending"
  )
end

#fail(job, error = "") ⇒ Object

Record a failed attempt (mirrors the lite backend + the Python Mongo adapter). Increments attempts; while attempts < max_retries the job is re-queued to pending (visible again immediately, or after retry_backoff), otherwise it is dead-lettered. The Queue/Job lifecycle expects fail() to route the requeue here — previously MongoBackend had no fail(), so job.fail() degraded to in-memory bookkeeping and never touched Mongo.



155
156
157
158
159
160
161
162
163
164
165
166
167
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 155

def fail(job, error = "")
  job.attempts += 1
  if job.attempts >= @max_retries
    collection.find_one_and_update(
      { _id: job.id },
      { "$set" => { status: "dead", topic: "#{job.topic}.dead_letter",
                    error: error, reserved_at: nil } },
      upsert: true
    )
  else
    requeue_with_error(job, error)
  end
end

#purge(topic, status) ⇒ Object 



220
221
222
223
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 220

def purge(topic, status)
  result = collection.delete_many(topic: topic, status: status.to_s)
  result.deleted_count
end

#reclaim_expired(topic, max_retries) ⇒ Object

Return reservations whose visibility window expired (at-least-once).

A message left “processing” with available_at <= now had a consumer die before acknowledging. Each is atomically flipped back to “pending” with attempts incremented (so the next dequeue re-delivers it); once attempts >= max_retries it is dead-lettered instead. Returns the number reclaimed. Disabled when visibility_timeout <= 0.



103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 103

def reclaim_expired(topic, max_retries)
  return 0 if @visibility_timeout.nil? || @visibility_timeout <= 0

  reclaimed = 0
  loop do
    now = Time.now.utc
    doc = collection.find_one_and_update(
      { topic: topic, status: "processing", available_at: { "$lte" => now } },
      { "$set" => { status: "pending", available_at: now, reserved_at: nil },
        "$inc" => { attempts: 1 } },
      sort: { available_at: 1 },
      return_document: :after
    )
    break unless doc

    reclaimed += 1
    next if (doc["attempts"] || 0) < max_retries

    # Out of retries — move it to the dead-letter queue and remove the
    # original so it is not re-delivered.
    collection.insert_one(
      _id: "#{doc["_id"]}.dead_letter",
      topic: "#{topic}.dead_letter",
      payload: doc["payload"],
      status: "dead",
      priority: doc["priority"] || 0,
      attempts: doc["attempts"] || 0,
      error: "reservation timed out - consumer did not acknowledge within the visibility timeout",
      created_at: Time.now.utc
    )
    collection.delete_one(_id: doc["_id"], topic: topic)
  end
  reclaimed
end

#requeue(message) ⇒ Object 



183
184
185
186
187
188
189
190
191
192
193
194
195
196
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 183

def requeue(message)
  # Reset available_at so the requeued job is visible again right away (or
  # after retry_backoff) and clear reserved_at. dequeue() pushed
  # available_at out to the reservation expiry; leaving it there stranded a
  # requeued job for the full visibility window instead of retrying it on
  # the next pop().
  collection.find_one_and_update(
    { _id: message.id },
    { "$set" => { status: "pending", reserved_at: nil,
                  available_at: requeue_available_at(@retry_backoff) },
      "$inc" => { attempts: 1 } },
    upsert: true
  )
end

#resolve_visibility_timeout(option) ⇒ Object 



47
48
49
50
51
52
53
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 47

def resolve_visibility_timeout(option)
  return option.to_f unless option.nil?

  Float(ENV.fetch("TINA4_QUEUE_VISIBILITY_TIMEOUT", "300"))
rescue ArgumentError, TypeError
  300.0
end

#retry(job, delay_seconds: 0) ⇒ Object

Explicit re-queue requested by the caller (job.retry()). Always re-enqueues regardless of the retry limit — a manual override, distinct from the automatic fail() path. Increments attempts, clears the error.



172
173
174
175
176
177
178
179
180
181
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 172

def retry(job, delay_seconds: 0)
  job.attempts += 1
  backoff = delay_seconds.to_f > 0 ? delay_seconds.to_f : @retry_backoff
  collection.find_one_and_update(
    { _id: job.id },
    { "$set" => { status: "pending", error: nil, reserved_at: nil,
                  available_at: requeue_available_at(backoff) } },
    upsert: true
  )
end

#retry_failed(topic, max_retries: 3) ⇒ Object 



225
226
227
228
229
230
231
232
233
234
235
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 225

def retry_failed(topic, max_retries: 3)
  result = collection.update_many(
    { topic: topic, status: "failed", attempts: { "$lt" => max_retries } },
    # Reset available_at so re-queued failed jobs are visible again — they
    # were reserved with available_at in the future at dequeue. Clear
    # reserved_at too. (Same Bug B reason as requeue/fail.)
    { "$set" => { status: "pending", error: nil, reserved_at: nil,
                  available_at: requeue_available_at(@retry_backoff) } }
  )
  result.modified_count
end

#size(topic) ⇒ Object 



206
207
208
 
# File 'lib/tina4/queue_backends/mongo_backend.rb', line 206

def size(topic)
  collection.count_documents(topic: topic, status: "pending")
end