Class: Cosmo::Stream::Processor

Inherits:
Processor show all
Defined in:
lib/cosmo/stream/processor.rb,
sig/cosmo/stream/processor.rbs

Constant Summary

Constants inherited from Processor

Processor::STREAMS_PAUSED_IDLE_SLEEP, Processor::STREAM_EMPTY_BACKOFF_MAX, Processor::STREAM_PAUSED_RECHECK_TTL

Instance Attribute Summary

Attributes inherited from Processor

#consumers

Instance Method Summary collapse

Methods inherited from Processor

#client, #consumer_state, #fetch, #initialize, #lock, run, #run, #run_loop, #running?, #schedule_loop, #scheduler?, #stop, #stopwatch, #work_loop

Constructor Details

This class inherits a constructor from Cosmo::Processor

Instance Method Details

#dynamic_config ⇒ Array[Hash[Symbol, untyped]]

Returns:

  • (Array[Hash[Symbol, untyped]]) 


51
52
53
 
# File 'lib/cosmo/stream/processor.rb', line 51

def dynamic_config
  Config.internal[:streams]&.map { _1.default_options.merge(class: _1) }.to_a
end

#fetch_subjects(config) ⇒ Object

Parameters:

  • config (Hash[Symbol, untyped])

Returns:

  • (Object) 


64
65
66
 
# File 'lib/cosmo/stream/processor.rb', line 64

def fetch_subjects(config)
  config.dig(:consumer, :subjects)
end

#fetch_timeout(config) ⇒ Float

Parameters:

  • config (Hash[Symbol, untyped])

Returns:

  • (Float) 


68
69
70
71
72
73
74
75
76
 
# File 'lib/cosmo/stream/processor.rb', line 68

def fetch_timeout(config)
  timeout = config[:fetch_timeout].to_f
  if timeout <= 0
    Logger.warn "Ignoring `fetch_timeout: #{timeout}` (causes high CPU usage) with #{Data::DEFAULTS[:fetch_timeout]}s instead"
    timeout = Data::DEFAULTS[:fetch_timeout].to_f
  end

  timeout
end

#process(messages, processor) ⇒ void

This method returns an undefined value.

rubocop:disable Metrics/AbcSize

Parameters:

  • messages (Array[untyped])
  • processor (Object) 


20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/cosmo/stream/processor.rb', line 20

def process(messages, processor) # rubocop:disable Metrics/AbcSize
  metadata = messages.last.metadata
  serializer = processor.class.default_options.dig(:publisher, :serializer)
  messages = messages.map { Message.new(_1, serializer:) }

  Logger.with(
    seq_stream: metadata.sequence.stream,
    seq_consumer: metadata.sequence.consumer,
    num_pending: metadata.num_pending,
    timestamp: metadata.timestamp
  ) { Logger.info "start" }

  sw = stopwatch
  processor.process(messages)
  Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "done" }
rescue StandardError => e
  Logger.debug e
  Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
rescue Exception # rubocop:disable Lint/RescueException
  Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
  raise
end

#setup ⇒ void

This method returns an undefined value.



8
9
10
11
12
13
14
15
16
17
18
 
# File 'lib/cosmo/stream/processor.rb', line 8

def setup
  @configs ||= []
  @configs = static_config + dynamic_config

  if @options[:processors]
    pattern = Regexp.new(@options[:processors].map { "\\b#{_1}\\b" }.join("|"))
    @configs.select! { _1[:class].name.match?(pattern) }
  end

  @configs.each { @consumers << subscribe(nil, _1) }
end

#static_config ⇒ Array[Hash[Symbol, untyped]]

Returns:

  • (Array[Hash[Symbol, untyped]]) 


43
44
45
46
47
48
49
 
# File 'lib/cosmo/stream/processor.rb', line 43

def static_config
  Config.dig(:consumers, :streams)&.filter_map do |config|
    next unless (klass = Utils::String.safe_constantize(config[:class]))

    config.merge(class: klass)
  end.to_a
end

#subscribe(_stream_name, config) ⇒ [untyped, Hash[Symbol, untyped], untyped]

Parameters:

  • stream_name (Symbol, nil)
  • config (Hash[Symbol, untyped])

Returns:

  • ([untyped, Hash[Symbol, untyped], untyped]) 


55
56
57
58
59
60
61
62
 
# File 'lib/cosmo/stream/processor.rb', line 55

def subscribe(_stream_name, config)
  processor = config[:class].new
  subjects = config.dig(:consumer, :subjects)
  deliver_policy = Config.deliver_policy(config[:start_position])
  consumer_config, consumer_name = config.values_at(:consumer, :consumer_name)
  subscription = client.subscribe(subjects, consumer_name, consumer_config.merge(deliver_policy))
  [subscription, config, processor]
end