Class: Lemans::Bench::Verifier

Inherits:
Section
  • Object
show all
Defined in:
lib/lemans/bench.rb

Overview

The verifier section: how a finished trial is verified, in the same sandbox the agent worked in, after Trial closes its network.

Constant Summary collapse

DEFAULT_COMMAND = 
"if [ -x /tests/verify ]; then exec /tests/verify; " \
"elif [ -f /tests/verification_test.rb ]; then exec ruby -report-lemans /tests/verification_test.rb; " \
"else exec bash /tests/test.sh; fi"
VALIDATED = 
%i[timeout_sec setup preverify command restore_paths logs_dir reward_path].freeze

Instance Method Summary collapse

Methods inherited from Section

#validate!

Constructor Details

#initialize(config) ⇒ Verifier

Returns a new instance of Verifier.



173
 
# File 'lib/lemans/bench.rb', line 173

def initialize(config) = super(config, "verifier")

Instance Method Details

#command ⇒ Object 



180
 
# File 'lib/lemans/bench.rb', line 180

def command = fetch("command", DEFAULT_COMMAND)

#logs_dir ⇒ Object 



193
194
195
196
197
198
 
# File 'lib/lemans/bench.rb', line 193

def logs_dir
  fetch("logs_dir", "/logs/verifier").tap do |dir|
    raise ConfigError, "verifier.logs_dir must be an absolute path, got #{dir.inspect}" unless
      dir.start_with?("/")
  end
end

#preverify ⇒ Object 



182
183
184
185
186
187
 
# File 'lib/lemans/bench.rb', line 182

def preverify
  self["preverify"].tap do |command|
    raise ConfigError, "#{dotted("preverify")} must be a command string, got #{command.inspect}" unless
      command.nil? || command.is_a?(String)
  end
end

#restore_paths ⇒ Object

The graded surfaces restored from the pre-agent snapshot before the command runs; a task may override the list in its frontmatter.



191
 
# File 'lib/lemans/bench.rb', line 191

def restore_paths = RestorePaths.call(self["restore"], label: dotted("restore"))

#reward_path ⇒ Object

Derived, never declared: the reward lands beside the logs that justify it.



201
 
# File 'lib/lemans/bench.rb', line 201

def reward_path = "#{logs_dir.chomp("/")}/reward.txt"

#setup ⇒ Object

These run after the network closes, so anything they need must already be in the image.



178
 
# File 'lib/lemans/bench.rb', line 178

def setup = commands("setup")

#timeout_sec ⇒ Object 



175
 
# File 'lib/lemans/bench.rb', line 175

def timeout_sec = seconds("timeout", default: "10m")