Class: Lemans::Bench

Inherits:
Object
  • Object
show all
Defined in:
lib/lemans/bench.rb

Overview

The frozen run profile, written once at the root of a bench: everything that must be identical across every trial. Task files carry only what differs.

Defined Under Namespace

Classes: Agent, Environment, Resources, Revision, Section, Verifier

Constant Summary collapse

DEFAULT_FILENAME = 
"bench.yml"
DEFAULT_RESOURCES = 
Resources.new(cpus: 2, memory_mb: 2048, storage_mb: 5120)
VERIFICATION_DIR = 
"verification"

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config, path:) ⇒ Bench

Returns a new instance of Bench.



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
 
# File 'lib/lemans/bench.rb', line 219

def initialize(config, path:)
  @path = Pathname(path)
  @root = @path.dirname
  @config = config

  @environment = Environment.new(section("environment"))
  @environment.validate!
  @agent = Agent.new(section("agent"))
  @agent.validate!
  @verifier = Verifier.new(section("verifier"))
  @verifier.validate!

  @files = SetupFiles.call(@config["files"], root: @root, label: @path)
  # Resolved once: an hours-long run reports the bench it started from, not later tree drift.
  @revision = Revision.detect(@root)
  digest
  freeze
end

Instance Attribute Details

#agent ⇒ Object (readonly)

Returns the value of attribute agent.



204
205
206
 
# File 'lib/lemans/bench.rb', line 204

def agent
  @agent
end

#environment ⇒ Object (readonly)

Returns the value of attribute environment.



204
205
206
 
# File 'lib/lemans/bench.rb', line 204

def environment
  @environment
end

#path ⇒ Object (readonly)

Returns the value of attribute path.



204
205
206
 
# File 'lib/lemans/bench.rb', line 204

def path
  @path
end

#revision ⇒ Object (readonly)

Returns the value of attribute revision.



204
205
206
 
# File 'lib/lemans/bench.rb', line 204

def revision
  @revision
end

#root ⇒ Object (readonly)

Returns the value of attribute root.



204
205
206
 
# File 'lib/lemans/bench.rb', line 204

def root
  @root
end

#verifier ⇒ Object (readonly)

Returns the value of attribute verifier.



204
205
206
 
# File 'lib/lemans/bench.rb', line 204

def verifier
  @verifier
end

Class Method Details

.load(path) ⇒ Object 



206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/lemans/bench.rb', line 206

def self.load(path)
  path = Pathname(path)
  path = path.join(DEFAULT_FILENAME) if path.directory?
  raise ConfigError, "no #{DEFAULT_FILENAME} at #{path}" unless path.file?

  config = YAML.safe_load_file(path, aliases: true) || {}
  raise ConfigError, "#{path}: bench.yml must be a mapping of sections" unless config.is_a?(Hash)

  new(config, path: path)
rescue Psych::Exception => e
  raise ConfigError, "#{path}: #{e.message}"
end

Instance Method Details

#digest ⇒ Object

Recorded on every result: two trials are only comparable under the same profile, and the bench's own files count as profile.



240
241
242
 
# File 'lib/lemans/bench.rb', line 240

def digest
  @digest ||= Digest::SHA256.hexdigest(JSON.generate([@config, file_digests]))[0, 16]
end

#file_digests ⇒ Object

The only thing a result carries that pins the bytes of the scripts a trial ran. Shared verification files count: they grade every trial.



248
249
250
251
252
253
254
255
 
# File 'lib/lemans/bench.rb', line 248

def file_digests
  @file_digests ||= begin
    shared = verification_files.map { |absolute, _| absolute.relative_path_from(root) }
    (@files.values.flatten + shared).map(&:to_s).sort.uniq.to_h do |path|
      [path, Digest::SHA256.file(root.join(path)).hexdigest]
    end
  end.freeze
end

#setup_files(phase) ⇒ Object 



244
 
# File 'lib/lemans/bench.rb', line 244

def setup_files(phase) = @files.fetch(phase.to_sym, [])

#tasks ⇒ Object

Raises:



268
269
270
271
272
 
# File 'lib/lemans/bench.rb', line 268

def tasks
  raise ConfigError, "no tasks directory at #{tasks_dir}" unless tasks_dir.directory?

  tasks_dir.children.select(&:directory?).sort.map { Task.load(_1, bench: self) }
end

#tasks_dir ⇒ Object 



266
 
# File 'lib/lemans/bench.rb', line 266

def tasks_dir = root.join(@config.fetch("tasks", "tasks"))

#verification_files ⇒ Object 



259
260
261
262
263
264
 
# File 'lib/lemans/bench.rb', line 259

def verification_files
  dir = root.join(VERIFICATION_DIR)
  return [] unless dir.directory?

  dir.glob("**/*", File::FNM_DOTMATCH).select(&:file?).map { [_1, _1.relative_path_from(dir).to_s] }
end