Class: Mergify::RSpec::FlakyDetector

Inherits:
Object
  • Object
show all
Defined in:
lib/mergify/rspec/flaky_detection.rb

Overview

Manages intelligent test rerunning with budget constraints for flaky detection. rubocop:disable Metrics/ClassLength

Defined Under Namespace

Classes: TestMetrics

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(token:, url:, full_repository_name:, mode:) ⇒ FlakyDetector

Returns a new instance of FlakyDetector.



67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 67

def initialize(token:, url:, full_repository_name:, mode:)
  @token = token
  @url = url
  @full_repository_name = full_repository_name
  @mode = mode
  @metrics = {}
  @over_length_tests = Set.new
  @tests_to_process = []
  @budget = 0.0

  fetch_context
  validate!
end

Instance Attribute Details

#budget ⇒ Object (readonly)

Returns the value of attribute budget.



65
66
67
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 65

def budget
  @budget
end

#mode ⇒ Object (readonly)

Returns the value of attribute mode.



65
66
67
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 65

def mode
  @mode
end

#tests_to_process ⇒ Object (readonly)

Returns the value of attribute tests_to_process.



65
66
67
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 65

def tests_to_process
  @tests_to_process
end

Instance Method Details

#fill_metrics_from_report(test_id, phase, duration, status) ⇒ Object

rubocop:disable Metrics/MethodLength



109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 109

def fill_metrics_from_report(test_id, phase, duration, status)
  if status == :skipped
    @metrics.delete(test_id)
    return
  end

  return unless @tests_to_process.include?(test_id)

  if test_id.length > @context[:max_test_name_length]
    @over_length_tests.add(test_id)
    return
  end

  # Only initialize metrics when the first phase is "setup"
  return if !@metrics.key?(test_id) && phase != 'setup'

  @metrics[test_id] ||= TestMetrics.new
  @metrics[test_id].fill_from_report(phase, duration, status)
end

#last_rerun_for_test?(test_id) ⇒ Boolean

Returns:

  • (Boolean) 


162
163
164
165
166
167
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 162

def last_rerun_for_test?(test_id)
  return false unless @metrics.key?(test_id)

  metrics = @metrics[test_id]
  metrics.will_exceed_deadline? || metrics.rerun_count >= @context[:max_test_execution_count]
end

#make_report ⇒ Object

rubocop:disable Metrics/MethodLength,Metrics/AbcSize



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 174

def make_report
  lines = []
  lines << 'Mergify Flaky Detection Report'
  lines << "  Mode        : #{@mode}"
  lines << "  Budget      : #{format('%.2f', @budget)}s"
  lines << "  Budget used : #{format('%.2f', budget_used)}s"
  lines << "  Tests tracked: #{@metrics.size}"
  lines << ''

  @metrics.each do |test_id, m|
    lines << "  #{test_id}"
    lines << "    Reruns       : #{m.rerun_count}"
    lines << "    Initial dur  : #{format('%.3f', m.initial_duration)}s"
    lines << "    Total dur    : #{format('%.3f', m.total_duration)}s"
    lines << "    Timeout warn : #{m.prevented_timeout}" if m.prevented_timeout
  end

  lines << '' unless @over_length_tests.empty?
  @over_length_tests.each do |id|
    lines << "  WARNING: test name too long (skipped): #{id[0, 80]}..."
  end

  lines.join("\n")
end

#prepare_for_session(test_ids) ⇒ Object

rubocop:disable Metrics/MethodLength,Metrics/AbcSize



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 82

def prepare_for_session(test_ids)
  existing = Set.new(@context[:existing_test_names])
  unhealthy = Set.new(@context[:unhealthy_test_names])

  @tests_to_process =
    if @mode == 'new'
      test_ids.reject { |id| existing.include?(id) }
    else
      test_ids.select { |id| unhealthy.include?(id) }
    end

  budget_ratio = if @mode == 'new'
                   @context[:budget_ratio_for_new_tests]
                 else
                   @context[:budget_ratio_for_unhealthy_tests]
                 end

  mean_duration_s = @context[:existing_tests_mean_duration_ms] / 1000.0
  existing_count = @context[:existing_test_names].size
  min_budget_s = @context[:min_budget_duration_ms] / 1000.0

  ratio_budget = budget_ratio * mean_duration_s * existing_count
  @budget = [ratio_budget, min_budget_s].max
end

#rerunning_test?(test_id) ⇒ Boolean

rubocop:enable Metrics/MethodLength

Returns:

  • (Boolean) 


130
131
132
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 130

def rerunning_test?(test_id)
  @metrics.key?(test_id) && @metrics[test_id].rerun_count >= 1
end

#set_test_deadline(test_id, timeout: nil) ⇒ Object 



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 138

def set_test_deadline(test_id, timeout: nil)
  return unless @metrics.key?(test_id)

  remaining_tests = [remaining_tests_count, 1].max
  per_test_budget = remaining_budget / remaining_tests

  allocated =
    if timeout
      [per_test_budget, timeout * 0.9].min
    else
      per_test_budget
    end

  @metrics[test_id].deadline = Time.now.to_f + allocated
end

#test_metrics(test_id) ⇒ Object 



169
170
171
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 169

def test_metrics(test_id)
  @metrics[test_id]
end

#test_rerun?(test_id) ⇒ Boolean

Returns:

  • (Boolean) 


134
135
136
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 134

def test_rerun?(test_id)
  @metrics.key?(test_id) && @metrics[test_id].rerun_count > 1
end

#test_too_slow?(test_id) ⇒ Boolean

Returns:

  • (Boolean) 


154
155
156
157
158
159
160
 
# File 'lib/mergify/rspec/flaky_detection.rb', line 154

def test_too_slow?(test_id)
  return false unless @metrics.key?(test_id)

  metrics = @metrics[test_id]
  min_exec = @context[:min_test_execution_count]
  (metrics.initial_duration * min_exec) > metrics.remaining_time
end