Module: Perfgate::Comparison::StatisticalMetricDecision

Defined in:
lib/perfgate/comparison/statistical_metric_decision.rb

Overview

Statistical decision path used for continuous, noisy metrics (duration, sql_duration, allocations): combines a practical threshold check with a one-sided Mann-Whitney U test, and downgrades a would-be fail to a warn when the metric is noisy (spec sections 16.3-16.5).

Class Method Summary collapse

Class Method Details

.breaches?(change, percent_threshold, minimum_absolute) ⇒ Boolean

Returns:

  • (Boolean) 


70
71
72
 
# File 'lib/perfgate/comparison/statistical_metric_decision.rb', line 70

def breaches?(change, percent_threshold, minimum_absolute)
  change[:change_percent].abs >= percent_threshold && change[:absolute_change].abs >= minimum_absolute
end

.call(metric, baseline_samples, candidate_samples, config) ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/perfgate/comparison/statistical_metric_decision.rb', line 16

def call(metric, baseline_samples, candidate_samples, config)
  change = MetricChange.summarize(metric, baseline_samples, candidate_samples)
  thresholds = threshold_for(metric, config)
  p_value = Statistics::MannWhitneyU.one_sided_p(baseline_samples, candidate_samples)

  significance = significance_flags(change, thresholds, p_value, config)
  noisy = MetricChange.noisy?(change[:baseline_summary], config)
  decision = decide(significance[:exceeds_failure], significance[:statistically_significant],
                    significance[:practically_significant], noisy)

  MetricChange.result(change, confidence: (1 - p_value).round(4),
                              practically_significant: significance[:practically_significant], noisy: noisy,
                              decision: decision)
end

.decide(exceeds_failure, statistically_significant, practically_significant, noisy) ⇒ Object 



41
42
43
44
45
46
47
48
49
 
# File 'lib/perfgate/comparison/statistical_metric_decision.rb', line 41

def decide(exceeds_failure, statistically_significant, practically_significant, noisy)
  if exceeds_failure && statistically_significant
    noisy ? "warn" : "fail"
  elsif practically_significant || statistically_significant
    "warn"
  else
    "pass"
  end
end

.minimum_absolute_ns(raw) ⇒ Object 



64
65
66
67
68
 
# File 'lib/perfgate/comparison/statistical_metric_decision.rb', line 64

def minimum_absolute_ns(raw)
  return raw[:minimum_absolute_ms] * 1_000_000 if raw[:minimum_absolute_ms]

  raw[:minimum_absolute] || 0
end

.significance_flags(change, thresholds, p_value, config) ⇒ Object 



31
32
33
34
35
36
37
38
39
 
# File 'lib/perfgate/comparison/statistical_metric_decision.rb', line 31

def significance_flags(change, thresholds, p_value, config)
  alpha = 1 - config.comparison_confidence_level
  clears_floor = change[:absolute_change].abs >= thresholds[:minimum_absolute]
  {
    practically_significant: breaches?(change, thresholds[:warning_percent], thresholds[:minimum_absolute]),
    exceeds_failure: breaches?(change, thresholds[:failure_percent], thresholds[:minimum_absolute]),
    statistically_significant: clears_floor && change[:absolute_change].positive? && p_value < alpha
  }
end

.threshold_for(metric, config) ⇒ Object 



51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/perfgate/comparison/statistical_metric_decision.rb', line 51

def threshold_for(metric, config)
  raw = config.dig(:comparison, :practical_thresholds, metric.to_sym) || {}
  {
    warning_percent: raw[:warning_percent] || Float::INFINITY,
    failure_percent: raw[:failure_percent] || Float::INFINITY,
    # minimum_absolute_ms is configured in milliseconds, but samples
    # (and therefore change[:absolute_change]) are always in the
    # metric's raw unit, nanoseconds for duration/sql_duration - so
    # it must be converted before comparison.
    minimum_absolute: minimum_absolute_ns(raw)
  }
end