Class: Benchmark::IPS::Job

Inherits:
Object
  • Object
show all
Defined in:
lib/benchmark/ips/job.rb,
lib/benchmark/ips/job/entry.rb,
lib/benchmark/ips/job/multi_report.rb,
lib/benchmark/ips/job/stream_report.rb

Overview

Benchmark jobs.

Defined Under Namespace

Classes: Entry, MultiReport, StreamReport

Constant Summary collapse

MICROSECONDS_PER_100MS =

Microseconds per 100 millisecond.

100_000
MICROSECONDS_PER_SECOND =

Microseconds per second.

Timing::MICROSECONDS_PER_SECOND
MAX_TIME_SKEW =

The percentage of the expected runtime to allow before reporting a weird runtime

0.05
MAX_ITERATIONS =

Keep iterations below this to avoid overflow to 64-bit long or Bignum

1 << 30

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ Job

Instantiate the Benchmark::IPS::Job.



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/benchmark/ips/job.rb', line 72

def initialize opts={}
  @list = []
  @run_single = false
  @json_path = false
  @compare = false
  @compare_order = :fastest
  @held_path = nil
  @held_results = nil
  @max_width = 20 # automatically computed as entries are added

  @timing = Hash.new 1 # default to 1 in case warmup isn't run
  @full_report = Report.new

  # Default warmup and calculation time in seconds.
  @warmup = 2
  @time = 5
  @iterations = 1

  # Default statistical model
  @stats = :sd
  @confidence = 95

  @out = MultiReport.new(StreamReport.new(self))
end

Instance Attribute Details

#compare ⇒ Boolean (readonly)

Determining whether to run comparison utility.

Returns:

  • (Boolean) —

    true if needs to run compare.



21
22
23
 
# File 'lib/benchmark/ips/job.rb', line 21

def compare
  @compare
end

#confidence ⇒ Integer

Confidence.

Returns:

  • (Integer) 


53
54
55
 
# File 'lib/benchmark/ips/job.rb', line 53

def confidence
  @confidence
end

#full_report ⇒ Report (readonly)

Report object containing information about the run.

Returns:

  • (Report) —

    the report object.



29
30
31
 
# File 'lib/benchmark/ips/job.rb', line 29

def full_report
  @full_report
end

#hold ⇒ Boolean

Determining whether to hold results between Ruby invocations

Returns:

  • (Boolean) 


25
26
27
 
# File 'lib/benchmark/ips/job.rb', line 25

def hold
  @hold
end

#iterations ⇒ Integer

Warmup and calculation iterations.

Returns:

  • (Integer) 


45
46
47
 
# File 'lib/benchmark/ips/job.rb', line 45

def iterations
  @iterations
end

#list ⇒ Array<Entry> (readonly)

Two-element arrays, consisting of label and block pairs.

Returns:

  • (Array<Entry>) —

    list of entries



17
18
19
 
# File 'lib/benchmark/ips/job.rb', line 17

def list
  @list
end

#max_width ⇒ Integer (readonly)

The maximum label width

Returns:

  • (Integer) 


57
58
59
 
# File 'lib/benchmark/ips/job.rb', line 57

def max_width
  @max_width
end

#stats ⇒ Object

Statistics model.

Returns:

  • (Object) 


49
50
51
 
# File 'lib/benchmark/ips/job.rb', line 49

def stats
  @stats
end

#time ⇒ Integer

Calculation time setter and getter (in seconds).

Returns:

  • (Integer) 


41
42
43
 
# File 'lib/benchmark/ips/job.rb', line 41

def time
  @time
end

#timing ⇒ Hash (readonly)

Storing Iterations in time period.

Returns:

  • (Hash) 


33
34
35
 
# File 'lib/benchmark/ips/job.rb', line 33

def timing
  @timing
end

#warmup ⇒ Integer

Warmup time setter and getter (in seconds).

Returns:

  • (Integer) 


37
38
39
 
# File 'lib/benchmark/ips/job.rb', line 37

def warmup
  @warmup
end

Instance Method Details

#all_results_have_been_run? ⇒ Boolean

Returns:

  • (Boolean) 


249
250
251
 
# File 'lib/benchmark/ips/job.rb', line 249

def all_results_have_been_run?
  @full_report.entries.size == @list.size
end

#clear_held_results ⇒ Object 



253
254
255
 
# File 'lib/benchmark/ips/job.rb', line 253

def clear_held_results
  File.delete @held_path if File.exist?(@held_path)
end

#compare!(order: :fastest) ⇒ Object

Run comparison utility.



130
131
132
133
 
# File 'lib/benchmark/ips/job.rb', line 130

def compare!(order: :fastest)
  @compare = true
  @compare_order = order
end

#compare? ⇒ Boolean

Return true if job needs to be compared.

Returns:

  • (Boolean) —

    Need to compare?



125
126
127
 
# File 'lib/benchmark/ips/job.rb', line 125

def compare?
  @compare
end

#config(opts) ⇒ Object

Job configuration options, set @warmup and @time.

Parameters:

  • opts (Hash) —

    a customizable set of options

  • iterations (Hash) —

    a customizable set of options

Options Hash (opts):

  • :warmup (Integer) —

    Warmup time.

  • :time (Integer) —

    Calculation time.



101
102
103
104
105
106
107
108
109
 
# File 'lib/benchmark/ips/job.rb', line 101

def config opts
  @warmup = opts[:warmup] if opts[:warmup]
  @time = opts[:time] if opts[:time]
  @iterations = opts[:iterations] if opts[:iterations]
  @stats = opts[:stats] if opts[:stats]
  @confidence = opts[:confidence] if opts[:confidence]
  self.quiet = opts[:quiet] if opts.key?(:quiet)
  self.suite = opts[:suite] if opts[:suite]
end

#create_report(label, measured_us, iter, samples, cycles) ⇒ Report::Entry

Create report by add entry to @full_report.

Parameters:

  • label (String) —

    Report item label.

  • measured_us (Integer) —

    Measured time in microsecond.

  • iter (Integer) —

    Iterations.

  • samples (Array<Float>) —

    Sampled iterations per second.

  • cycles (Integer) —

    Number of Cycles.

Returns:



402
403
404
 
# File 'lib/benchmark/ips/job.rb', line 402

def create_report(label, measured_us, iter, samples, cycles)
  @full_report.add_entry label, measured_us, iter, samples, cycles
end

#create_stats(samples, measured_us, iterations) ⇒ Object 



374
375
376
377
378
379
380
381
382
383
 
# File 'lib/benchmark/ips/job.rb', line 374

def create_stats(samples, measured_us, iterations)
  case @stats
    when :sd
      Stats::SD.new(samples, measured_us, iterations)
    when :bootstrap
      Stats::Bootstrap.new(samples, @confidence)
    else
      raise "unknown stats #{@stats}"
  end
end

#cycles_per_100ms(time_msec, iters) ⇒ Integer

Calculate the cycles needed to run for approx 100ms, given the number of iterations to run the given time.

Parameters:

  • time_msec (Float) —

    Each iteration’s time in ms.

  • iters (Integer) —

    Iterations.

Returns:

  • (Integer) —

    Cycles per 100ms.



201
202
203
204
 
# File 'lib/benchmark/ips/job.rb', line 201

def cycles_per_100ms time_msec, iters
  cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
  cycles <= 0 ? 1 : cycles
end

#generate_json ⇒ Object

Generate json from @full_report.



391
392
393
 
# File 'lib/benchmark/ips/job.rb', line 391

def generate_json
  @full_report.generate_json @json_path if json?
end

#hold!(held_path) ⇒ Object

Hold after each iteration.

Parameters:

  • held_path (String) —

    File name to store hold file.



143
144
145
146
 
# File 'lib/benchmark/ips/job.rb', line 143

def hold!(held_path)
  @held_path = held_path
  @run_single = true
end

#hold? ⇒ Boolean

Return true if results are held while multiple Ruby invocations

Returns:

  • (Boolean) —

    Need to hold results between multiple Ruby invocations?



137
138
139
 
# File 'lib/benchmark/ips/job.rb', line 137

def hold?
  !!@held_path
end

#item(label = "", str = nil, &blk) ⇒ Object Also known as: report

Registers the given label and block pair in the job list.

Parameters:

  • label (String) (defaults to: "") —

    Label of benchmarked code.

  • str (String) (defaults to: nil) —

    Code to be benchmarked.

  • blk (Proc) —

    Code to be benchmarked.

Raises:

  • (ArgumentError) —

    Raises if str and blk are both present.

  • (ArgumentError) —

    Raises if str and blk are both absent.



181
182
183
184
185
186
187
188
189
190
191
192
193
 
# File 'lib/benchmark/ips/job.rb', line 181

def item(label="", str=nil, &blk) # :yield:
  if blk and str
    raise ArgumentError, "specify a block and a str, but not both"
  end

  action = str || blk
  raise ArgumentError, "no block or string" unless action

  @max_width = label.size if label.size > @max_width

  @list.push Entry.new(label, action)
  self
end

#iterations_per_sec(cycles, time_us) ⇒ Float

Calculate the iterations per second given the number of cycles run and the time in microseconds that elapsed.

Parameters:

  • cycles (Integer) —

    Cycles.

  • time_us (Integer) —

    Time in microsecond.

Returns:

  • (Float) —

    Iteration per second.



219
220
221
 
# File 'lib/benchmark/ips/job.rb', line 219

def iterations_per_sec cycles, time_us
  MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
end

#json!(path = "data.json") ⇒ Object

Generate json to given path, defaults to “data.json”.



171
172
173
 
# File 'lib/benchmark/ips/job.rb', line 171

def json!(path="data.json")
  @json_path = path
end

#json? ⇒ Boolean

Return true if job needs to generate json.

Returns:

  • (Boolean) —

    Need to generate json?



166
167
168
 
# File 'lib/benchmark/ips/job.rb', line 166

def json?
  !!@json_path
end

#load_held_results ⇒ Object 



223
224
225
226
227
228
229
230
231
232
 
# File 'lib/benchmark/ips/job.rb', line 223

def load_held_results
  return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
  require "json"
  @held_results = {}
  JSON.load(IO.read(@held_path)).each do |result|
    @held_results[result['item']] = result
    create_report(result['item'], result['measured_us'], result['iter'],
                  create_stats(result['samples'], result['measured_us'], result['iter']), result['cycles'])
  end
end

#quiet ⇒ Boolean

Silence output

Returns:

  • (Boolean) 


61
62
63
 
# File 'lib/benchmark/ips/job.rb', line 61

def quiet
  @out.quiet?
end

#quiet=(val) ⇒ Object 



111
112
113
114
115
116
117
 
# File 'lib/benchmark/ips/job.rb', line 111

def quiet=(val)
  if val # remove instances of StreamReport
    @out.quiet!
  else # ensure there is an instance of StreamReport
    @out << StreamReport.new(self) if @out.quiet?
  end
end

#run ⇒ Object 



257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
 
# File 'lib/benchmark/ips/job.rb', line 257

def run
  if @warmup && @warmup != 0 then
    @out.start_warming
    @iterations.times do
      run_warmup
    end
  end

  @out.start_running

  @iterations.times do |n|
    run_benchmark
  end

  @out.footer
end

#run_benchmark ⇒ Object

Run calculation.



322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
 
# File 'lib/benchmark/ips/job.rb', line 322

def run_benchmark
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @out.running item.label, @time

    Timing.clean_env

    iter = 0

    measurements_us = []

    # Running this number of cycles should take around 100ms.
    cycles = @timing[item]

    target = Timing.add_second Timing.now, @time

    begin
      before = Timing.now
      item.call_times cycles
      after = Timing.now

      # If for some reason the timing said this took no time (O_o)
      # then ignore the iteration entirely and start another.
      iter_us = Timing.time_us before, after
      next if iter_us <= 0.0

      iter += cycles

      measurements_us << iter_us
    end while Timing.now < target

    final_time = before

    measured_us = measurements_us.inject(:+)

    samples = measurements_us.map { |time_us|
      iterations_per_sec cycles, time_us
    }

    rep = create_report(item.label, measured_us, iter, create_stats(samples, measured_us, iter), cycles)

    if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
      rep.show_total_time!
    end

    @out.add_report rep, caller(1).first

    break if run_single?
  end
end

#run_comparison ⇒ Object

Run comparison of entries in @full_report.



386
387
388
 
# File 'lib/benchmark/ips/job.rb', line 386

def run_comparison
  @full_report.run_comparison(@compare_order) if compare?
end

#run_single? ⇒ Boolean

Return true if items are to be run one at a time. For the traditional hold, this is true

Returns:

  • (Boolean) —

    Run just a single item?



160
161
162
 
# File 'lib/benchmark/ips/job.rb', line 160

def run_single?
  @run_single
end

#run_warmup ⇒ Object

Run warmup.



275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
 
# File 'lib/benchmark/ips/job.rb', line 275

def run_warmup
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @out.warming item.label, @warmup

    Timing.clean_env

    # Run for up to half of the configured warmup time with an increasing
    # number of cycles to reduce overhead and improve accuracy.
    # This also avoids running with a constant number of cycles, which a
    # JIT might speculate on and then have to recompile in #run_benchmark.
    before = Timing.now
    target = Timing.add_second before, @warmup / 2.0

    cycles = 1
    begin
      t0 = Timing.now
      item.call_times cycles
      t1 = Timing.now
      warmup_iter = cycles
      warmup_time_us = Timing.time_us(t0, t1)

      # If the number of cycles would go outside the 32-bit signed integers range
      # then exit the loop to avoid overflows and start the 100ms warmup runs
      break if cycles >= MAX_ITERATIONS
      cycles *= 2
    end while Timing.now + warmup_time_us * 2 < target

    per_100ms = cycles_per_100ms warmup_time_us, warmup_iter
    # Not [per_100ms, MAX_ITERATIONS].min as that can promote the result to long
    cycles = per_100ms > MAX_ITERATIONS ? MAX_ITERATIONS : per_100ms
    @timing[item] = cycles

    # Run for the remaining of warmup in a similar way as #run_benchmark.
    target = Timing.add_second before, @warmup
    while Timing.now + MICROSECONDS_PER_100MS < target
      item.call_times cycles
    end

    @out.warmup_stats warmup_time_us, @timing[item]

    break if run_single?
  end
end

#save!(held_path) ⇒ Object

Save interim results. Similar to hold, but all reports are run The report label must change for each invocation. One way to achieve this is to include the version in the label.

Parameters:

  • held_path (String) —

    File name to store hold file.



152
153
154
155
 
# File 'lib/benchmark/ips/job.rb', line 152

def save!(held_path)
  @held_path = held_path
  @run_single = false
end

#save_held_results ⇒ Object 



234
235
236
237
238
239
240
241
242
243
244
245
246
247
 
# File 'lib/benchmark/ips/job.rb', line 234

def save_held_results
  return unless @held_path
  require "json"
  data = full_report.entries.map { |e|
    {
      'item' => e.label,
      'measured_us' => e.microseconds,
      'iter' => e.iterations,
      'samples' => e.samples,
      'cycles' => e.measurement_cycle
    }
  }
  IO.write(@held_path, JSON.generate(data) << "\n")
end

#suite ⇒ Benchmark::IPS::MultiReport

Suite

Returns:

  • (Benchmark::IPS::MultiReport) 


67
68
69
 
# File 'lib/benchmark/ips/job.rb', line 67

def suite
  @out
end

#suite=(suite) ⇒ Object 



119
120
121
 
# File 'lib/benchmark/ips/job.rb', line 119

def suite=(suite)
  @out << suite
end

#time_us(before, after) ⇒ Float

Calculate the time difference of before and after in microseconds.

Parameters:

  • before (Time) —

    time.

  • after (Time) —

    time.

Returns:

  • (Float) —

    Time difference of before and after.



210
211
212
 
# File 'lib/benchmark/ips/job.rb', line 210

def time_us before, after
  (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
end