Class: Lemans::Config

Inherits:
Object
  • Object
show all
Defined in:
lib/lemans/config.rb,
lib/lemans/config/agent.rb,
lib/lemans/config/setup.rb,
lib/lemans/config/revision.rb,
lib/lemans/config/verifier.rb,
lib/lemans/config/conversion.rb,
lib/lemans/config/image_spec.rb,
lib/lemans/config/environment.rb,
lib/lemans/config/tree_digest.rb,
lib/lemans/config/network_policy.rb

Overview

Benchmark configuration and task definitions container.

Defined Under Namespace

Modules: Conversion, TreeDigest Classes: Agent, Environment, ImageSpec, NetworkPolicy, Revision, Setup, Verifier

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(root = Pathname("./"), config_path: Pathname("./bench.yml"), version: nil, tasks_dir: "tasks", setup: nil, agent: Agent.new("miniswen", "openrouter/z-ai/glm-5.2"), environment: Environment.new, verifier: Verifier.new) ⇒ Config

Returns a new instance of Config.



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/lemans/config.rb', line 53

def initialize(root = Pathname("./"), config_path: Pathname("./bench.yml"), version: nil, tasks_dir: "tasks",
               setup: nil, agent: Agent.new("miniswen", "openrouter/z-ai/glm-5.2"),
               environment: Environment.new, verifier: Verifier.new)
  @root = root
  @config_path = config_path
  @version = version
  @tasks_dir = root.join(tasks_dir)
  @setup = setup || Setup.new
  @agent = agent
  @environment = environment
  @environment.dockerfile = root.join(@environment.dockerfile) if @environment.dockerfile
  @environment.dockerfile ||= default_dockerfile unless @environment.image
  @verifier = verifier
  @verifier.root = root
  @concurrency = 4
  @attempts = 1
  @backend = @environment.backend
  @tasks = parse_tasks
end

Instance Attribute Details

#agent ⇒ Object (readonly)

Nested configs



15
16
17
 
# File 'lib/lemans/config.rb', line 15

def agent
  @agent
end

#attempts ⇒ Object (readonly)

Returns the value of attribute attempts.



10
11
12
 
# File 'lib/lemans/config.rb', line 10

def attempts
  @attempts
end

#backend ⇒ Object (readonly)

Returns the value of attribute backend.



10
11
12
 
# File 'lib/lemans/config.rb', line 10

def backend
  @backend
end

#concurrency ⇒ Object (readonly)

Returns the value of attribute concurrency.



10
11
12
 
# File 'lib/lemans/config.rb', line 10

def concurrency
  @concurrency
end

#config_path ⇒ Object (readonly)

Returns the value of attribute config_path.



10
11
12
 
# File 'lib/lemans/config.rb', line 10

def config_path
  @config_path
end

#environment ⇒ Object (readonly)

Nested configs



15
16
17
 
# File 'lib/lemans/config.rb', line 15

def environment
  @environment
end

#root ⇒ Object (readonly)

Returns the value of attribute root.



10
11
12
 
# File 'lib/lemans/config.rb', line 10

def root
  @root
end

#setup ⇒ Object (readonly)

Nested configs



15
16
17
 
# File 'lib/lemans/config.rb', line 15

def setup
  @setup
end

#tasks ⇒ Object (readonly)

Returns the value of attribute tasks.



10
11
12
 
# File 'lib/lemans/config.rb', line 10

def tasks
  @tasks
end

#tasks_dir ⇒ Object (readonly)

Returns the value of attribute tasks_dir.



10
11
12
 
# File 'lib/lemans/config.rb', line 10

def tasks_dir
  @tasks_dir
end

#verifier ⇒ Object (readonly)

Nested configs



15
16
17
 
# File 'lib/lemans/config.rb', line 15

def verifier
  @verifier
end

#version ⇒ Object (readonly)

Returns the value of attribute version.



10
11
12
 
# File 'lib/lemans/config.rb', line 10

def version
  @version
end

Class Method Details

.load_file(path) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/lemans/config.rb', line 18

def load_file(path)
  raise ConfigError, "bench directory not found: #{path}" unless File.directory?(path)

  config_name = %w[bench.yaml bench.yml].find { File.file?(File.join(path, it)) }

  raise ConfigError, "bench.yml not found" unless config_name

  config_path = Pathname(File.join(path, config_name))

  contents = YAML.safe_load_file(config_path.to_s, aliases: true) || {}
  raise ConfigError, "#{path}: #{config_name} must be a mapping of sections" unless contents.is_a?(Hash)

  root = Pathname(path)

  sections = {}
  sections[:version] = contents["version"]
  sections[:tasks_dir] = contents["tasks"]
  sections[:setup] = Setup.from_config(contents["setup"], root:)
  sections[:agent] = Agent.from_config(contents["agent"])
  sections[:environment] = Environment.from_config(contents["environment"])
  sections[:verifier] = Verifier.from_config(contents["verifier"], root:)
  sections.compact!

  # Older benches declared environment-phase commands under `environment.setup`.
  if (commands = contents.dig("environment", "setup"))
    sections[:setup] ||= Setup.new
    sections[:setup].commands = Array(commands) + sections[:setup].commands
  end

  new(root, config_path:, **sections)
rescue Psych::Exception => e
  raise ConfigError, "#{path}: #{e.message}"
end

Instance Method Details

#agent_name ⇒ Object 



81
 
# File 'lib/lemans/config.rb', line 81

def agent_name = agent.name

#digest ⇒ Object

Digest captures the state of the config and verification files. Used for resuming runs mostly



92
93
94
95
96
97
98
99
100
 
# File 'lib/lemans/config.rb', line 92

def digest
  @digest ||= begin
    sha = Digest::SHA256.new
    sha << TreeDigest.call(root.join(Verifier::VERIFICATION_DIR))
    sha << TreeDigest.call(environment.dockerfile.dirname) if environment.dockerfile
    sha << Digest::SHA256.file(config_path.to_s).hexdigest if config_path.file?
    sha.hexdigest[0, 16]
  end
end

#load_options(agent: nil, model: nil, attempts: nil, concurrency: nil, backend: nil) ⇒ Object 



73
74
75
76
77
78
79
 
# File 'lib/lemans/config.rb', line 73

def load_options(agent: nil, model: nil, attempts: nil, concurrency: nil, backend: nil, **)
  @agent.name = agent if agent
  @agent.models = Array(model) if model
  @attempts = attempts if attempts
  @concurrency = concurrency if concurrency
  @backend = environment.backend = backend if backend
end

#models ⇒ Object 



83
 
# File 'lib/lemans/config.rb', line 83

def models = agent.models

#revision ⇒ Object

Resolved once: an hours-long run reports the bench it started from, not later tree drift.



87
88
89
 
# File 'lib/lemans/config.rb', line 87

def revision
  @revision ||= Revision.detect(root)
end