Class: ElasticGraph::IndexerAutoscalerLambda::ConcurrencyScaler

Inherits:
Object
  • Object
show all
Defined in:
lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb

Constant Summary collapse

MINIMUM_CONCURRENCY = 
2

Instance Method Summary collapse

Constructor Details

#initialize(datastore_core:, sqs_client:, lambda_client:, cloudwatch_client:) ⇒ ConcurrencyScaler

Returns a new instance of ConcurrencyScaler.



15
16
17
18
19
20
21
 
# File 'lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb', line 15

def initialize(datastore_core:, sqs_client:, lambda_client:, cloudwatch_client:)
  @logger = datastore_core.logger
  @datastore_core = datastore_core
  @sqs_client = sqs_client
  @lambda_client = lambda_client
  @cloudwatch_client = cloudwatch_client
end

Instance Method Details

#tune_indexer_concurrency(queue_urls:, min_cpu_target:, max_cpu_target:, maximum_concurrency:, required_free_storage_in_mb:, indexer_function_name:, cluster_name:) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/elastic_graph/indexer_autoscaler_lambda/concurrency_scaler.rb', line 25

def tune_indexer_concurrency(
  queue_urls:,
  min_cpu_target:,
  max_cpu_target:,
  maximum_concurrency:,
  required_free_storage_in_mb:,
  indexer_function_name:,
  cluster_name:
)
  queue_attributes = get_queue_attributes(queue_urls)
  queue_arns = queue_attributes.fetch(:queue_arns)
  num_messages = queue_attributes.fetch(:total_messages)

  details_logger = DetailsLogger.new(
    logger: @logger,
    queue_arns: queue_arns,
    queue_urls: queue_urls,
    min_cpu_target: min_cpu_target,
    max_cpu_target: max_cpu_target,
    num_messages: num_messages
  )

  new_target_concurrency =
    if num_messages.positive?
      lowest_node_free_storage_in_mb = get_lowest_node_free_storage_in_mb(cluster_name)

      cpu_utilization = get_max_cpu_utilization
      cpu_midpoint = (max_cpu_target + min_cpu_target) / 2.0

      current_concurrency = get_concurrency(indexer_function_name)

      if current_concurrency.nil?
        details_logger.log_unset
        nil
      elsif lowest_node_free_storage_in_mb < required_free_storage_in_mb
        details_logger.log_pause(
          lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
          required_free_storage_in_mb: required_free_storage_in_mb
        )
        MINIMUM_CONCURRENCY
      elsif cpu_utilization < min_cpu_target
        increase_factor = (cpu_midpoint / cpu_utilization).clamp(0.0, 1.5)
        (current_concurrency * increase_factor).round.tap do |new_concurrency|
          details_logger.log_increase(
            cpu_utilization: cpu_utilization,
            lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
            required_free_storage_in_mb: required_free_storage_in_mb,
            current_concurrency: current_concurrency,
            new_concurrency: new_concurrency
          )
        end
      elsif cpu_utilization > max_cpu_target
        decrease_factor = cpu_utilization / cpu_midpoint - 1
        (current_concurrency - (current_concurrency * decrease_factor)).round.tap do |new_concurrency|
          details_logger.log_decrease(
            cpu_utilization: cpu_utilization,
            lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
            required_free_storage_in_mb: required_free_storage_in_mb,
            current_concurrency: current_concurrency,
            new_concurrency: new_concurrency
          )
        end
      else
        details_logger.log_no_change(
          cpu_utilization: cpu_utilization,
          lowest_node_free_storage_in_mb: lowest_node_free_storage_in_mb,
          required_free_storage_in_mb: required_free_storage_in_mb,
          current_concurrency: current_concurrency
        )
        current_concurrency
      end
    else
      details_logger.log_reset
      MINIMUM_CONCURRENCY
    end

  if new_target_concurrency && new_target_concurrency != current_concurrency
    update_concurrency(
      indexer_function_name: indexer_function_name,
      concurrency: new_target_concurrency,
      maximum_concurrency: maximum_concurrency
    )
  end
end