Module: Corkscrews::Validate

Defined in:
lib/corkscrews/validate.rb

Defined Under Namespace

Classes: Result

Constant Summary collapse

ROOT = 
File.expand_path("../..", __dir__)
BENCHMARK_ROOT = 
File.join(ROOT, "validate", "benchmarks")

Class Method Summary collapse

Class Method Details

.benchmark_environment(settings, manifest) ⇒ Object 



337
338
339
340
341
342
 
# File 'lib/corkscrews/validate.rb', line 337

def benchmark_environment(settings, manifest)
  manifest.fetch("env", {}).transform_values(&:to_s).merge(
    "CS_BENCH_ITERATIONS" => settings.fetch(:iterations).to_s,
    "CORKSCREWS_ROUND_MS" => settings.fetch(:round_ms).to_s
  )
end

.find_target(aggregate, manifest, target) ⇒ Object 



317
318
319
320
321
322
323
324
325
326
327
 
# File 'lib/corkscrews/validate.rb', line 317

def find_target(aggregate, manifest, target)
  aggregate.fetch(:targets).find do |candidate|
    if target.fetch("kind").to_s == "wait"
      candidate[:kind] == "wait" && candidate[:name] == target.fetch("name")
    else
      candidate[:kind] == "line" &&
        File.expand_path(candidate[:file]) == File.expand_path(target.fetch("file"), manifest.fetch("_dir")) &&
        candidate[:line].to_i == target.fetch("line").to_i
    end
  end
end

.load_manifests ⇒ Object 



64
65
66
67
68
 
# File 'lib/corkscrews/validate.rb', line 64

def load_manifests
  Dir[File.join(BENCHMARK_ROOT, "*", "manifest.yml")]
    .sort
    .map { |path| YAML.load_file(path).merge("_path" => path, "_dir" => File.dirname(path)) }
end

.measure_actual(manifest, settings:) ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/corkscrews/validate.rb', line 128

def measure_actual(manifest, settings:)
  base = measure_variant(manifest, "base.rb", settings: settings)
  optimized = measure_variant(manifest, "optimized.rb", settings: settings)
  improvement = (Statistics.mean(optimized[:rates]) / Statistics.mean(base[:rates])) - 1.0

  {
    base: base,
    optimized: optimized,
    improvement_pct: improvement * 100.0
  }
end

.measure_variant(manifest, file_name, settings:) ⇒ Object 



140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/corkscrews/validate.rb', line 140

def measure_variant(manifest, file_name, settings:)
  path = File.join(manifest.fetch("_dir"), file_name)
  rates = Array.new(settings.fetch(:repeat)) do
    env = benchmark_environment(settings, manifest).merge("CS_BENCH_JSON" => "1")
    stdout, stderr, status = Open3.capture3(env, ruby, "-I#{File.join(ROOT, "lib")}", path, chdir: ROOT)
    raise Error, "benchmark #{file_name} failed: #{stderr}" unless status.success?

    payload = JSON.parse(stdout.lines.last)
    payload.fetch("progress").to_f / (payload.fetch("elapsed_ns").to_f / 1_000_000_000)
  end

  { rates: rates, mean_rate: Statistics.mean(rates), rate_ci: Statistics.bootstrap_mean_ci(rates) }
end

.profile_benchmark(manifest, settings:, output:) ⇒ Object

Raises:



105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/corkscrews/validate.rb', line 105

def profile_benchmark(manifest, settings:, output:)
  bench = File.join(manifest.fetch("_dir"), "base.rb")
  command = [
    ruby,
    "-I#{File.join(ROOT, "lib")}",
    File.join(ROOT, "exe", "corkscrews"),
    "run",
    "--repeat", settings.fetch(:repeat).to_s,
    "--output", output,
    "--sample-period-ms", settings.fetch(:sample_period_ms).to_s,
    "--targets", manifest.fetch("targets", "lines"),
    "--",
    ruby,
    "-I#{File.join(ROOT, "lib")}",
    bench
  ]
  env = benchmark_environment(settings, manifest)
  stdout, stderr, status = Open3.capture3(env, *command, chdir: ROOT)
  raise Error, "profile failed: #{stderr}\n#{stdout}" unless status.success?

  Analysis.load(output)
end

.ruby ⇒ Object 



344
345
346
 
# File 'lib/corkscrews/validate.rb', line 344

def ruby
  RbConfig.ruby
end

.run_all(quick: false, benchmark: nil) ⇒ Object

Raises:



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/corkscrews/validate.rb', line 44

def run_all(quick: false, benchmark: nil)
  manifests = load_manifests
  manifests.select! { |manifest| manifest.fetch("id") == benchmark } if benchmark
  raise Error, "unknown benchmark: #{benchmark}" if manifests.empty?

  checks = []
  profiles = {}
  Dir.mktmpdir("corkscrews-validate") do |tmpdir|
    manifests.each do |manifest|
      benchmark_checks, profile = run_benchmark(manifest, tmpdir: tmpdir, quick: quick)
      checks.concat(benchmark_checks)
      profiles[manifest.fetch("id")] = profile
    end
    checks << t5_statistics_check
    checks << t6_native_attribution_check(manifests, profiles: profiles, benchmark: benchmark)
  end

  Result.new(checks: checks)
end

.run_benchmark(manifest, tmpdir:, quick:) ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/corkscrews/validate.rb', line 70

def run_benchmark(manifest, tmpdir:, quick:)
  id = manifest.fetch("id")
  settings = run_settings(manifest, quick: quick)
  profile_path = File.join(tmpdir, "#{id}.corkscrews.ndjson")
  profile = profile_benchmark(manifest, settings: settings, output: profile_path)
  actual = measure_actual(manifest, settings: settings)

  checks = [
    t1_prediction_check(manifest, profile, actual),
    t2_actionability_check(manifest),
    t3_decoy_check(manifest, profile),
    t4_overhead_check(manifest, profile, actual)
  ]
  [checks, profile]
end

.run_settings(manifest, quick:) ⇒ Object 



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/corkscrews/validate.rb', line 86

def run_settings(manifest, quick:)
  run = manifest.fetch("run")
  if quick
    {
      repeat: [run.fetch("repeat").to_i, 3].min,
      iterations: [run.fetch("iterations").to_i, 8_000].min,
      sample_period_ms: run.fetch("quick_sample_period_ms", 0.5).to_f,
      round_ms: run.fetch("round_ms", 40).to_f
    }
  else
    {
      repeat: run.fetch("repeat").to_i,
      iterations: run.fetch("iterations").to_i,
      sample_period_ms: run.fetch("sample_period_ms", 1.0).to_f,
      round_ms: run.fetch("round_ms", 40).to_f
    }
  end
end

.t1_prediction_check(manifest, profile, actual) ⇒ Object 



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/corkscrews/validate.rb', line 154

def t1_prediction_check(manifest, profile, actual)
  # Paper basis: Coz SOSP'15 Section 4 evaluates causal profiles by
  # comparing predicted speedup with actual optimized variants.
  # Paper URL: https://arxiv.org/abs/1608.03676
  truth = manifest.fetch("truth")
  target = truth.fetch("target")
  target_kind = target.fetch("kind").to_s
  speedup_pct = (truth.fetch("optimized_speedup_of_target").to_f * 100).round
  speedup_pct -= speedup_pct % 5

  curve = target_curve(profile, manifest, target)
  predicted = curve&.find { |point| point[:speedup_pct] == speedup_pct }
  tolerance = manifest.fetch("acceptance").fetch("t1_mae_pct").to_f
  error = predicted ? (predicted[:improvement_pct] - actual.fetch(:improvement_pct)).abs : Float::INFINITY

  {
    id: "#{manifest.fetch("id")}:T1",
    ok: predicted && error <= tolerance,
    predicted_improvement_pct: predicted && predicted[:improvement_pct],
    actual_improvement_pct: actual.fetch(:improvement_pct),
    error_pct: error,
    tolerance_pct: tolerance,
    target: target_label(manifest, target)
  }
end

.t2_actionability_check(manifest) ⇒ Object 



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/corkscrews/validate.rb', line 180

def t2_actionability_check(manifest)
  # Paper basis: Mytkowicz et al., "Evaluating the Accuracy of Java
  # Profilers" (PLDI'10, DOI 10.1145/1806596.1806618), frames profile
  # quality around whether profiler guidance is actionable.
  # Paper URL: https://doi.org/10.1145/1806596.1806618
  variants = manifest.fetch("variants", [])
  return { id: "#{manifest.fetch("id")}:T2", ok: true, skipped: true } if variants.empty?

  predicted = variants.map { |variant| variant.fetch("predicted_rank").to_i }
  actual = variants.map { |variant| variant.fetch("actual_rank").to_i }
  tau = Statistics.kendall_tau(predicted, actual)

  {
    id: "#{manifest.fetch("id")}:T2",
    ok: tau >= manifest.fetch("acceptance").fetch("t2_min_tau", 0.6).to_f,
    kendall_tau: tau
  }
end

.t3_decoy_check(manifest, profile) ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/corkscrews/validate.rb', line 199

def t3_decoy_check(manifest, profile)
  # Paper basis: Coz SOSP'15 Figures 1-2 show hot code that is not
  # optimization-relevant; this check pins that false-hotspot case.
  # Paper URL: https://arxiv.org/abs/1608.03676
  decoys = manifest.dig("truth", "decoys") || []
  return { id: "#{manifest.fetch("id")}:T3", ok: true, skipped: true } if decoys.empty?

  max_effect = manifest.fetch("acceptance").fetch("t3_decoy_max_effect_pct", 5.0).to_f
  aggregate = profile.aggregate
  results = decoys.map do |decoy|
    target = find_target(aggregate, manifest, decoy)
    point = target&.fetch(:curve)&.find { |entry| entry[:speedup_pct] == 25 }
    effect = point ? point[:improvement_pct].abs : 0.0
    {
      target: target_label(manifest, decoy),
      observed_share_pct: target ? target[:sample_share].to_f * 100.0 : 0.0,
      causal_share_pct: target ? target[:causal_share].to_f * 100.0 : 0.0,
      effect_pct: effect,
      ok: effect <= max_effect
    }
  end

  {
    id: "#{manifest.fetch("id")}:T3",
    ok: results.all? { |result| result[:ok] },
    max_effect_pct: max_effect,
    decoys: results
  }
end

.t4_overhead_check(manifest, profile, actual) ⇒ Object 



229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
 
# File 'lib/corkscrews/validate.rb', line 229

def t4_overhead_check(manifest, profile, actual)
  # Paper basis: Coz SOSP'15 Section 4 reports profiling overhead;
  # Mytkowicz et al. PLDI'10 also treats observer effects as a source
  # of profiler inaccuracy.
  # Paper URLs: https://arxiv.org/abs/1608.03676
  # https://doi.org/10.1145/1806596.1806618
  progress_name = manifest.fetch("progress_point")
  profiled_rate = profile.aggregate.fetch(:progress).dig(progress_name, :mean_rate).to_f
  baseline_rate = actual.fetch(:base).fetch(:mean_rate)
  overhead_pct = baseline_rate.positive? ? ((baseline_rate - profiled_rate) / baseline_rate) * 100.0 : 0.0
  tolerance = manifest.fetch("acceptance").fetch("t4_overhead_pct").to_f

  {
    id: "#{manifest.fetch("id")}:T4",
    ok: overhead_pct <= tolerance,
    profiled_rate: profiled_rate,
    baseline_rate: baseline_rate,
    overhead_pct: overhead_pct,
    tolerance_pct: tolerance
  }
end

.t5_statistics_check ⇒ Object 



251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
 
# File 'lib/corkscrews/validate.rb', line 251

def t5_statistics_check
  # Paper basis: Kalibera & Jones, "Quantifying Performance Changes
  # with Effect Size Confidence Intervals" (arXiv:2007.10899),
  # motivates confidence intervals for performance effect sizes.
  # Paper URL: https://arxiv.org/abs/2007.10899
  values = [9.7, 10.1, 10.4, 9.9, 10.2, 10.0, 10.3, 9.8]
  ci = Statistics.bootstrap_mean_ci(values, iterations: 500)
  mean = Statistics.mean(values)
  monotonic = Statistics.monotonic_regression([
    { speedup_pct: 0, improvement_pct: 0.0 },
    { speedup_pct: 5, improvement_pct: 3.0 },
    { speedup_pct: 10, improvement_pct: 2.5 },
    { speedup_pct: 15, improvement_pct: 5.0 }
  ])
  nondecreasing = monotonic.each_cons(2).all? { |left, right| left[:improvement_pct] <= right[:improvement_pct] }

  {
    id: "T5",
    ok: ci[0] <= mean && mean <= ci[1] && nondecreasing,
    mean: mean,
    ci: ci,
    monotonic_points: monotonic
  }
end

.t6_native_attribution_check(manifests, profiles:, benchmark:) ⇒ Object 



276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
 
# File 'lib/corkscrews/validate.rb', line 276

def t6_native_attribution_check(manifests, profiles:, benchmark:)
  # Paper basis: Scalene OSDI'23 Section 2 separates interpreter,
  # native, and system execution; this check verifies native-call
  # evidence is attributed back to the Ruby call site.
  # Paper URL: https://www.usenix.org/system/files/osdi23-berger.pdf
  if benchmark && benchmark != "b08_native_attribution"
    return { id: "T6", ok: true, skipped: true }
  end

  manifest = manifests.find { |entry| entry.fetch("id") == "b08_native_attribution" }
  return { id: "T6", ok: false, native_benchmark_present: false } unless manifest

  profile = profiles.fetch("b08_native_attribution", nil)
  aggregate = profile&.aggregate || {}
  target = aggregate.empty? ? nil : find_target(aggregate, manifest, manifest.fetch("truth").fetch("target"))
  native = aggregate.fetch(:native, {})
  native_samples = native.fetch(:samples, 0).to_i
  target_hits = native.fetch(:target_hits, 0).to_i

  {
    id: "T6",
    ok: target && target.fetch(:samples).to_i.positive? && native_samples.positive? && target_hits.positive?,
    native_benchmark_present: true,
    target_samples: target&.fetch(:samples).to_i,
    native_samples: native_samples,
    native_target_hits: target_hits,
    target: target_label(manifest, manifest.fetch("truth").fetch("target"))
  }
end

.target_curve(profile, manifest, target) ⇒ Object 



306
307
308
309
310
311
312
313
314
315
 
# File 'lib/corkscrews/validate.rb', line 306

def target_curve(profile, manifest, target)
  if target.fetch("kind").to_s == "wait"
    profile.target_curve(kind: "wait", name: target.fetch("name"))
  else
    profile.target_curve(
      file: File.expand_path(target.fetch("file"), manifest.fetch("_dir")),
      line: target.fetch("line").to_i
    )
  end
end

.target_label(manifest, target) ⇒ Object 



329
330
331
332
333
334
335
 
# File 'lib/corkscrews/validate.rb', line 329

def target_label(manifest, target)
  if target.fetch("kind").to_s == "wait"
    "wait:#{target.fetch("name")}"
  else
    "#{File.expand_path(target.fetch("file"), manifest.fetch("_dir"))}:#{target.fetch("line")}"
  end
end