Class: CI::Queue::Configuration

Inherits:
Object
  • Object
show all
Defined in:
lib/ci/queue/configuration.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(timeout: 30, build_id: nil, worker_id: nil, max_requeues: 0, requeue_tolerance: 0, namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil, grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil, max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil, queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil, export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil, heartbeat_max_test_duration: nil, lazy_load: false, lazy_load_stream_batch_size: nil, lazy_load_streaming_timeout: nil, lazy_load_test_helpers: nil, skip_stale_tests: false) ⇒ Configuration

Returns a new instance of Configuration.



55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/ci/queue/configuration.rb', line 55

def initialize(
  timeout: 30, build_id: nil, worker_id: nil, max_requeues: 0, requeue_tolerance: 0,
  namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
  grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
  max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
  queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
  export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil, heartbeat_max_test_duration: nil,
  lazy_load: false, lazy_load_stream_batch_size: nil, lazy_load_streaming_timeout: nil, lazy_load_test_helpers: nil,
  skip_stale_tests: false)
  @build_id = build_id
  @circuit_breakers = [CircuitBreaker::Disabled]
  @failure_file = failure_file
  @flaky_tests = flaky_tests
  @grind_count = grind_count
  @max_requeues = max_requeues
  @max_test_duration = max_test_duration
  @max_test_duration_percentile = max_test_duration_percentile
  @max_test_failed = max_test_failed
  @namespace = namespace
  @requeue_tolerance = requeue_tolerance
  @seed = seed
  @statsd_endpoint = statsd_endpoint
  @timeout = timeout
  @queue_init_timeout = queue_init_timeout
  @track_test_duration = track_test_duration
  @worker_id = worker_id
  self.max_consecutive_failures = max_consecutive_failures
  self.max_duration = max_duration
  @redis_ttl = redis_ttl
  @report_timeout = report_timeout
  @inactive_workers_timeout = inactive_workers_timeout
  @export_flaky_tests_file = export_flaky_tests_file
  @warnings_file = warnings_file
  @debug_log = debug_log
  @max_missed_heartbeat_seconds = max_missed_heartbeat_seconds
  @heartbeat_max_test_duration = heartbeat_max_test_duration
  @lazy_load = lazy_load
  @lazy_load_stream_batch_size = lazy_load_stream_batch_size || 5_000
  @lazy_load_streaming_timeout = lazy_load_streaming_timeout
  @lazy_load_test_helpers = lazy_load_test_helpers
  @skip_stale_tests = skip_stale_tests
end

Instance Attribute Details

#build_id ⇒ Object 



182
183
184
185
186
187
188
 
# File 'lib/ci/queue/configuration.rb', line 182

def build_id
  if namespace
    "#{namespace}:#{@build_id}"
  else
    @build_id
  end
end

#circuit_breakers ⇒ Object (readonly)

Returns the value of attribute circuit_breakers.



14
15
16
 
# File 'lib/ci/queue/configuration.rb', line 14

def circuit_breakers
  @circuit_breakers
end

#debug_log ⇒ Object

Returns the value of attribute debug_log.



8
9
10
 
# File 'lib/ci/queue/configuration.rb', line 8

def debug_log
  @debug_log
end

#export_flaky_tests_file ⇒ Object

Returns the value of attribute export_flaky_tests_file.



5
6
7
 
# File 'lib/ci/queue/configuration.rb', line 5

def export_flaky_tests_file
  @export_flaky_tests_file
end

#failing_test ⇒ Object

Returns the value of attribute failing_test.



6
7
8
 
# File 'lib/ci/queue/configuration.rb', line 6

def failing_test
  @failing_test
end

#failure_file ⇒ Object

Returns the value of attribute failure_file.



5
6
7
 
# File 'lib/ci/queue/configuration.rb', line 5

def failure_file
  @failure_file
end

#grind_count ⇒ Object

Returns the value of attribute grind_count.



5
6
7
 
# File 'lib/ci/queue/configuration.rb', line 5

def grind_count
  @grind_count
end

#heartbeat_max_test_duration ⇒ Object 



158
159
160
 
# File 'lib/ci/queue/configuration.rb', line 158

def heartbeat_max_test_duration
  @heartbeat_max_test_duration || (timeout * 10 if max_missed_heartbeat_seconds)
end

#inactive_workers_timeout ⇒ Object 



154
155
156
 
# File 'lib/ci/queue/configuration.rb', line 154

def inactive_workers_timeout
  @inactive_workers_timeout || timeout
end

#lazy_load ⇒ Object

Returns the value of attribute lazy_load.



10
11
12
 
# File 'lib/ci/queue/configuration.rb', line 10

def lazy_load
  @lazy_load
end

#lazy_load_stream_batch_size ⇒ Object

Returns the value of attribute lazy_load_stream_batch_size.



10
11
12
 
# File 'lib/ci/queue/configuration.rb', line 10

def lazy_load_stream_batch_size
  @lazy_load_stream_batch_size
end

#lazy_load_streaming_timeout ⇒ Object 



117
118
119
120
121
122
123
 
# File 'lib/ci/queue/configuration.rb', line 117

def lazy_load_streaming_timeout
  if @lazy_load_streaming_timeout && @lazy_load_streaming_timeout > 0
    @lazy_load_streaming_timeout
  else
    [queue_init_timeout, 300].max
  end
end

#lazy_load_test_helpers ⇒ Object

Returns the value of attribute lazy_load_test_helpers.



12
13
14
 
# File 'lib/ci/queue/configuration.rb', line 12

def lazy_load_test_helpers
  @lazy_load_test_helpers
end

#max_missed_heartbeat_seconds ⇒ Object

Returns the value of attribute max_missed_heartbeat_seconds.



8
9
10
 
# File 'lib/ci/queue/configuration.rb', line 8

def max_missed_heartbeat_seconds
  @max_missed_heartbeat_seconds
end

#max_requeues ⇒ Object

Returns the value of attribute max_requeues.



5
6
7
 
# File 'lib/ci/queue/configuration.rb', line 5

def max_requeues
  @max_requeues
end

#max_test_duration ⇒ Object

Returns the value of attribute max_test_duration.



7
8
9
 
# File 'lib/ci/queue/configuration.rb', line 7

def max_test_duration
  @max_test_duration
end

#max_test_duration_percentile ⇒ Object

Returns the value of attribute max_test_duration_percentile.



7
8
9
 
# File 'lib/ci/queue/configuration.rb', line 7

def max_test_duration_percentile
  @max_test_duration_percentile
end

#max_test_failed ⇒ Object

Returns the value of attribute max_test_failed.



8
9
10
 
# File 'lib/ci/queue/configuration.rb', line 8

def max_test_failed
  @max_test_failed
end

#namespace ⇒ Object

Returns the value of attribute namespace.



6
7
8
 
# File 'lib/ci/queue/configuration.rb', line 6

def namespace
  @namespace
end

#queue_init_timeout ⇒ Object 



109
110
111
 
# File 'lib/ci/queue/configuration.rb', line 109

def queue_init_timeout
  @queue_init_timeout || timeout
end

#redis_ttl ⇒ Object

Returns the value of attribute redis_ttl.



8
9
10
 
# File 'lib/ci/queue/configuration.rb', line 8

def redis_ttl
  @redis_ttl
end

#report_timeout ⇒ Object 



113
114
115
 
# File 'lib/ci/queue/configuration.rb', line 113

def report_timeout
  @report_timeout || timeout
end

#requeue_tolerance ⇒ Object

Returns the value of attribute requeue_tolerance.



6
7
8
 
# File 'lib/ci/queue/configuration.rb', line 6

def requeue_tolerance
  @requeue_tolerance
end

#seed ⇒ Object 



178
179
180
 
# File 'lib/ci/queue/configuration.rb', line 178

def seed
  @seed || build_id
end

#skip_stale_tests ⇒ Object

Returns the value of attribute skip_stale_tests.



13
14
15
 
# File 'lib/ci/queue/configuration.rb', line 13

def skip_stale_tests
  @skip_stale_tests
end

#statsd_endpoint ⇒ Object

Returns the value of attribute statsd_endpoint.



6
7
8
 
# File 'lib/ci/queue/configuration.rb', line 6

def statsd_endpoint
  @statsd_endpoint
end

#timeout ⇒ Object

Returns the value of attribute timeout.



5
6
7
 
# File 'lib/ci/queue/configuration.rb', line 5

def timeout
  @timeout
end

#track_test_duration ⇒ Object

Returns the value of attribute track_test_duration.



7
8
9
 
# File 'lib/ci/queue/configuration.rb', line 7

def track_test_duration
  @track_test_duration
end

#warnings_file ⇒ Object

Returns the value of attribute warnings_file.



8
9
10
 
# File 'lib/ci/queue/configuration.rb', line 8

def warnings_file
  @warnings_file
end

#worker_id ⇒ Object

Returns the value of attribute worker_id.



5
6
7
 
# File 'lib/ci/queue/configuration.rb', line 5

def worker_id
  @worker_id
end

Class Method Details

.from_env(env) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/ci/queue/configuration.rb', line 19

def from_env(env)
  lazy_load_value = env['CI_QUEUE_LAZY_LOAD']
  lazy_load = lazy_load_value && !lazy_load_value.strip.empty? && !%w(0 false).include?(lazy_load_value.strip.downcase)
  new(
    build_id: env['CIRCLE_BUILD_URL'] || env['BUILDKITE_BUILD_ID'] || env['TRAVIS_BUILD_ID'] || env['HEROKU_TEST_RUN_ID'] || env['SEMAPHORE_PIPELINE_ID'],
    worker_id: env['CIRCLE_NODE_INDEX'] || env['BUILDKITE_PARALLEL_JOB'] || env['CI_NODE_INDEX'] || env['SEMAPHORE_JOB_ID'],
    seed: env['CIRCLE_SHA1'] || env['BUILDKITE_COMMIT'] || env['TRAVIS_COMMIT'] || env['HEROKU_TEST_RUN_COMMIT_VERSION'] || env['SEMAPHORE_GIT_SHA'],
    flaky_tests: load_flaky_tests(env['CI_QUEUE_FLAKY_TESTS']),
    statsd_endpoint: env['CI_QUEUE_STATSD_ADDR'],
    redis_ttl: env['CI_QUEUE_REDIS_TTL']&.to_i ||  8 * 60 * 60,
    debug_log: env['CI_QUEUE_DEBUG_LOG'],
    max_requeues: env['CI_QUEUE_MAX_REQUEUES']&.to_i || 0,
    requeue_tolerance: env['CI_QUEUE_REQUEUE_TOLERANCE']&.to_f || 0,
    lazy_load: lazy_load || false,
    lazy_load_stream_batch_size: (env['CI_QUEUE_LAZY_LOAD_STREAM_BATCH_SIZE'] || env['CI_QUEUE_STREAM_BATCH_SIZE'])&.to_i,
    lazy_load_streaming_timeout: (env['CI_QUEUE_LAZY_LOAD_STREAM_TIMEOUT'] || env['CI_QUEUE_STREAM_TIMEOUT'])&.to_i,
    lazy_load_test_helpers: env['CI_QUEUE_LAZY_LOAD_TEST_HELPERS'] || env['CI_QUEUE_TEST_HELPERS'],
    skip_stale_tests: %w(1 true).include?(env['CI_QUEUE_SKIP_STALE_TESTS']&.strip&.downcase),
  )
end

.load_flaky_tests(path) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/ci/queue/configuration.rb', line 40

def load_flaky_tests(path)
  return [] unless path
  if ::File.extname(path) == ".xml"
    require 'rexml/document'
    REXML::Document.new(::File.read(path)).elements.to_a("//testcase").map do |element|
      "#{element.attributes['classname']}##{element.attributes['name']}"
    end.to_set
  else
    ::File.readlines(path).map(&:chomp).to_set
  end
rescue SystemCallError
  []
end

Instance Method Details

#flaky?(test) ⇒ Boolean

Returns:

  • (Boolean) 


174
175
176
 
# File 'lib/ci/queue/configuration.rb', line 174

def flaky?(test)
  @flaky_tests.include?(test.id)
end

#global_max_requeues(tests_count) ⇒ Object 



190
191
192
 
# File 'lib/ci/queue/configuration.rb', line 190

def global_max_requeues(tests_count)
  (tests_count * Float(requeue_tolerance)).ceil
end

#lazy_load_test_helper_paths ⇒ Object 



98
99
100
101
102
 
# File 'lib/ci/queue/configuration.rb', line 98

def lazy_load_test_helper_paths
  return [] unless @lazy_load_test_helpers

  @lazy_load_test_helpers.split(',').map(&:strip)
end

#max_consecutive_failures=(max) ⇒ Object 



162
163
164
165
166
 
# File 'lib/ci/queue/configuration.rb', line 162

def max_consecutive_failures=(max)
  if max
    @circuit_breakers << CircuitBreaker::MaxConsecutiveFailures.new(max_consecutive_failures: max)
  end
end

#max_duration=(duration) ⇒ Object 



168
169
170
171
172
 
# File 'lib/ci/queue/configuration.rb', line 168

def max_duration=(duration)
  if duration
    @circuit_breakers << CircuitBreaker::Timeout.new(duration: duration)
  end
end

#retry? ⇒ Boolean

Returns:

  • (Boolean) 


104
105
106
107
 
# File 'lib/ci/queue/configuration.rb', line 104

def retry?
  ENV.fetch("BUILDKITE_RETRY_COUNT", "0").to_i > 0 ||
    ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"
end

#stream_batch_size ⇒ Object

Backward-compatible aliases for existing callers.



126
127
128
 
# File 'lib/ci/queue/configuration.rb', line 126

def stream_batch_size
  lazy_load_stream_batch_size
end

#stream_batch_size=(value) ⇒ Object 



130
131
132
 
# File 'lib/ci/queue/configuration.rb', line 130

def stream_batch_size=(value)
  self.lazy_load_stream_batch_size = value
end

#streaming_timeout ⇒ Object 



134
135
136
 
# File 'lib/ci/queue/configuration.rb', line 134

def streaming_timeout
  lazy_load_streaming_timeout
end

#streaming_timeout=(value) ⇒ Object 



138
139
140
 
# File 'lib/ci/queue/configuration.rb', line 138

def streaming_timeout=(value)
  self.lazy_load_streaming_timeout = value
end

#test_helper_paths ⇒ Object 



150
151
152
 
# File 'lib/ci/queue/configuration.rb', line 150

def test_helper_paths
  lazy_load_test_helper_paths
end

#test_helpers ⇒ Object 



142
143
144
 
# File 'lib/ci/queue/configuration.rb', line 142

def test_helpers
  lazy_load_test_helpers
end

#test_helpers=(value) ⇒ Object 



146
147
148
 
# File 'lib/ci/queue/configuration.rb', line 146

def test_helpers=(value)
  self.lazy_load_test_helpers = value
end