Module: EndPointBlank::Writers::DelayedWriter

Included in:
ExceptionWriter, LogWriter, RequestWriter, ResponseWriter
Defined in:
lib/end_point_blank/writers/delayed_writer.rb

Overview

Drains a background send queue with a fixed pool of worker threads.

The queue is bounded (see MAX_QUEUE_SIZE) so that during an intake outage - when the worker threads can't drain as fast as the host application enqueues payloads - memory usage stays capped instead of growing without bound toward an OOM. When the queue is full, the oldest item is dropped to make room for the newest one, and a warning is logged, throttled so a sustained outage does not itself become a logging flood.

Constant Summary collapse

MAX_QUEUE_SIZE = 
1000
WARN_THROTTLE_SECONDS = 
30

Instance Method Summary collapse

Instance Method Details

#direct_writer ⇒ Object 



18
19
20
 
# File 'lib/end_point_blank/writers/delayed_writer.rb', line 18

def direct_writer
  @direct_writer ||= DirectWriter.new(url)
end

#enqueue(list) ⇒ Object 



60
61
62
63
 
# File 'lib/end_point_blank/writers/delayed_writer.rb', line 60

def enqueue(list)
  items = list.is_a?(Array) ? list : [list]
  items.each { |payload| enqueue_one(payload) }
end

#log_warning(message) ⇒ Object

Logs a drop warning. Overridable/stubbable in tests; routed through the pluggable EndPointBlank.logger seam (this lib is fire-and-forget and must never raise).



68
69
70
 
# File 'lib/end_point_blank/writers/delayed_writer.rb', line 68

def log_warning(message)
  EndPointBlank.logger.warn(message)
end

#pop_additional ⇒ Object 



54
55
56
57
58
 
# File 'lib/end_point_blank/writers/delayed_writer.rb', line 54

def pop_additional
  queue.pop(true)
rescue ThreadError
  nil
end

#queue ⇒ Object 



22
23
24
 
# File 'lib/end_point_blank/writers/delayed_writer.rb', line 22

def queue
  @queue ||= Queue.new
end

#start_threads ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/end_point_blank/writers/delayed_writer.rb', line 26

def start_threads
  @threads = []

  2.times do
    @threads << Thread.new do
      loop do
        payload = queue.pop
        payloads = [payload]
        while (payload = pop_additional) do
          payloads << payload
        end

        payloads.compact!
        while payloads.any? do
          list = payloads[0..5]
          response = direct_writer.write(list)
          if response.status < 299
            on_success(response) if respond_to?(:on_success)
          else
            on_failure(response) if respond_to?(:on_failure)
          end
          payloads = payloads - list
        end
      end
    end
  end
end