Module: PerformanceHelpers

Defined in:
lib/tasks/performance_helpers.rb

Defined Under Namespace

Modules: Base, Current

Class Method Summary collapse

Class Method Details

.clone_base_repo(base, performance_dir, script) ⇒ Object

Clone base branch into a temp dir and return its path



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/tasks/performance_helpers.rb', line 31

def clone_base_repo(base, performance_dir, script)
  puts "Cloning base #{base}..."
  safe_ref = base.gsub(/[^0-9A-Za-z._-]/, "-")
  clone_dir = File.join(performance_dir, "base-#{safe_ref}")
  FileUtils.rm_rf(clone_dir)

  repo_url, = ruby_exec("git config --get remote.origin.url")
  repo_url = repo_url.strip

  stdout, stderr, status = ruby_exec("git clone --branch #{safe_ref} --single-branch #{repo_url} #{clone_dir}")
  raise "git clone failed: #{stderr}\n#{stdout}" unless status.success?

  Dir.chdir(clone_dir) do
    stdout, stderr, status = ruby_exec("bundle install --quiet")
    raise "bundle install failed: #{stderr}\n#{stdout}" unless status.success?

    bench_copy_dir = File.join(clone_dir, "tmp", "performance")
    FileUtils.mkdir_p(bench_copy_dir)
    bench_copy = File.join(bench_copy_dir, "benchmark_runner.rb")
    File.write(bench_copy, File.read(script))
    load_into_namespace(Base, bench_copy)
  end
end

.compare_metrics(label, curr, base, threshold) ⇒ Object 



68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/tasks/performance_helpers.rb', line 68

def compare_metrics(label, curr, base, threshold)
  base_ips = base.fetch(:lower)
  curr_ips = curr.fetch(:upper)
  change = (curr_ips - base_ips) / base_ips.to_f

  {
    label: label,
    base_ips: base_ips,
    curr_ips: curr_ips,
    change: change,
    regressed: change < -threshold,
  }
end

.current_branch ⇒ Object 



25
26
27
28
 
# File 'lib/tasks/performance_helpers.rb', line 25

def current_branch
  stdout, = ruby_exec("git rev-parse --abbrev-ref HEAD")
  stdout.strip
end

.load_into_namespace(module_obj, file_path) ⇒ Object 



16
17
18
19
 
# File 'lib/tasks/performance_helpers.rb', line 16

def load_into_namespace(module_obj, file_path)
  content = File.read(file_path)
  module_obj.module_eval(content, file_path)
end

.log_regressions(regressions, threshold) ⇒ Object 



107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/tasks/performance_helpers.rb', line 107

def log_regressions(regressions, threshold)
  return if regressions.empty?

  puts "\nDetected regressions (< -#{(threshold * 100).round(2)}% IPS):"
  regressions.each do |regression|
    delta = regression[:delta_fraction]
    base_ips = regression[:base_ips]
    curr_ips = regression[:curr_ips]

    delta_str = delta ? format("%+0.2f%%", delta * 100) : "N/A"
    base_str = base_ips ? format("%.2f", base_ips) : "N/A"
    curr_str = curr_ips ? format("%.2f", curr_ips) : "N/A"

    puts format("%<label>30s: %<base>s -> %<curr>s IPS (change: %<delta>s)",
                label: regression[:label],
                base: base_str,
                curr: curr_str,
                delta: delta_str)
  end
end

.ruby_exec(cmd, env: {}) ⇒ Object 



21
22
23
 
# File 'lib/tasks/performance_helpers.rb', line 21

def ruby_exec(cmd, env: {})
  Open3.capture3(env, cmd)
end

.run_benchmarks(base_runner, current_runner, threshold, all_base, all_current) ⇒ Object 



55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/tasks/performance_helpers.rb', line 55

def run_benchmarks(base_runner, current_runner, threshold, all_base,
all_current)
  base_results = base_runner.run_benchmarks
  curr_results = current_runner.run_benchmarks

  all_base.merge!(base_results)
  all_current.merge!(curr_results)

  curr_results.each do |label, result|
    print_realtime_comparison(label, result, base_results[label], threshold)
  end
end

.summary_report(current_results, base_results, base, run_time, threshold) ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/tasks/performance_helpers.rb', line 82

def summary_report(current_results, base_results, base, run_time, threshold)
  summary = {
    run_time: run_time,
    threshold: threshold,
    branch: current_branch,
    base: base,
    regressions: [],
  }

  current_results.each do |label, metrics|
    cmp = compare_metrics(label, metrics, base_results[label], threshold)
    next unless cmp[:regressed]

    summary[:regressions] << {
      label: label,
      base_ips: cmp[:base_ips],
      curr_ips: cmp[:curr_ips],
      delta_fraction: cmp[:change],
    }
  end

  log_regressions(summary[:regressions], threshold)
  summary
end