Module: Perfgate::Comparison::MetricChange

Defined in:
lib/perfgate/comparison/metric_change.rb

Overview

Shared helpers for building a metric comparison result and for computing the baseline->candidate change that every decision path (statistical or deterministic) needs (spec section 14.2's workloads[].metrics entries).

Class Method Summary collapse

Class Method Details

.base_fields(metric) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/perfgate/comparison/metric_change.rb', line 18

def base_fields(metric)
  {
    "metric" => metric.to_s,
    "baseline_median" => nil,
    "candidate_median" => nil,
    "change_percent" => nil,
    "absolute_change" => nil,
    "confidence" => nil,
    "practically_significant" => false,
    "noisy" => false
  }
end

.build_change(metric, baseline_summary, candidate_summary, absolute_change, change_percent) ⇒ Object 



40
41
42
43
44
45
46
47
48
 
# File 'lib/perfgate/comparison/metric_change.rb', line 40

def build_change(metric, baseline_summary, candidate_summary, absolute_change, change_percent)
  {
    metric: metric.to_s,
    baseline_summary: baseline_summary,
    candidate_summary: candidate_summary,
    absolute_change: absolute_change,
    change_percent: change_percent
  }
end

.change_fields(change) ⇒ Object 



63
64
65
66
67
68
69
70
71
 
# File 'lib/perfgate/comparison/metric_change.rb', line 63

def change_fields(change)
  {
    "metric" => change[:metric],
    "baseline_median" => change[:baseline_summary]["median"],
    "candidate_median" => change[:candidate_summary]["median"],
    "change_percent" => change[:change_percent],
    "absolute_change" => change[:absolute_change]
  }
end

.inconclusive(metric) ⇒ Object 



14
15
16
 
# File 'lib/perfgate/comparison/metric_change.rb', line 14

def inconclusive(metric)
  base_fields(metric).merge("decision" => "inconclusive")
end

.noisy?(baseline_summary, config) ⇒ Boolean

A metric is "noisy" when its own baseline spread (MAD relative to its median) is large enough that a modest shift in medians could plausibly be explained by run-to-run variance alone (spec 16.5).

Returns:

  • (Boolean) 


76
77
78
79
80
81
 
# File 'lib/perfgate/comparison/metric_change.rb', line 76

def noisy?(baseline_summary, config)
  median = baseline_summary["median"]
  return false if median.zero?

  (baseline_summary["mad"] / median.to_f) > config.comparison_noise_ratio_threshold
end

.percent_change(baseline_median, absolute_change) ⇒ Object 



50
51
52
53
54
 
# File 'lib/perfgate/comparison/metric_change.rb', line 50

def percent_change(baseline_median, absolute_change)
  return 0.0 if baseline_median.zero?

  ((absolute_change / baseline_median.to_f) * 100).round(2)
end

.result(change, confidence:, practically_significant:, noisy:, decision:) ⇒ Object 



56
57
58
59
60
61
 
# File 'lib/perfgate/comparison/metric_change.rb', line 56

def result(change, confidence:, practically_significant:, noisy:, decision:)
  change_fields(change).merge(
    "confidence" => confidence, "practically_significant" => practically_significant,
    "noisy" => noisy, "decision" => decision
  )
end

.summarize(metric, baseline_samples, candidate_samples) ⇒ Object 



31
32
33
34
35
36
37
38
 
# File 'lib/perfgate/comparison/metric_change.rb', line 31

def summarize(metric, baseline_samples, candidate_samples)
  baseline_summary = Statistics::Summary.call(baseline_samples)
  candidate_summary = Statistics::Summary.call(candidate_samples)
  absolute_change = candidate_summary["median"] - baseline_summary["median"]
  change_percent = percent_change(baseline_summary["median"], absolute_change)

  build_change(metric, baseline_summary, candidate_summary, absolute_change, change_percent)
end