Class: LLMExperiment::Experiment

Inherits:
Object
  • Object
show all
Defined in:
lib/llm_experiment/experiment.rb

Overview

experiment.yml, loaded and validated. Unknown keys are errors: a typoed option that silently does nothing is how experiments measure the wrong thing.

Constant Summary collapse

TOP_KEYS = 
%w[name question pins agents conditions trial apps tasks].freeze
APP_KEYS = 
%w[ruby bundler database test_command db_prepare suite_ran_pattern
publish_transcripts branch_prefix neutralize].freeze
TASK_KEYS = 
%w[id app test_file impl_files].freeze
TRIAL_KEYS = 
%w[timeout_seconds memory cpus].freeze
AGENT_KEYS = 
%w[model].freeze
KNOWN_AGENTS = 
%w[claude codex].freeze
DATABASES = 
%w[sqlite3 postgresql].freeze
DEFAULT_SUITE_RAN = 
'\d+ runs?, \d+ assertions?'

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(data, root:) ⇒ Experiment

Returns a new instance of Experiment.

Raises:



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/llm_experiment/experiment.rb', line 59

def initialize(data, root:)
  @root = root
  reject_unknown!("experiment.yml", data, TOP_KEYS)
  @name = data["name"] or raise ConfigError, "experiment.yml needs a name"
  @question = data["question"].to_s
  @pins = Pins.resolve(data["pins"] || {})
  @agents = parse_agents(data["agents"] || {})
  @conditions = Array(data["conditions"]).map(&:to_s)
  raise ConfigError, "conditions must not be empty" if @conditions.empty?

  trial = data["trial"] || {}
  reject_unknown!("trial", trial, TRIAL_KEYS)
  @timeout_seconds = (trial["timeout_seconds"] || 900).to_i
  @memory = trial["memory"] || LLMExperiment.default_memory
  @cpus = (trial["cpus"] || LLMExperiment.default_cpus).to_s

  @apps = parse_apps(data["apps"] || {})
  @tasks = parse_tasks(data["tasks"] || [])
end

Instance Attribute Details

#agents ⇒ Object (readonly)

Returns the value of attribute agents.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def agents
  @agents
end

#apps ⇒ Object (readonly)

Returns the value of attribute apps.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def apps
  @apps
end

#conditions ⇒ Object (readonly)

Returns the value of attribute conditions.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def conditions
  @conditions
end

#cpus ⇒ Object (readonly)

Returns the value of attribute cpus.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def cpus
  @cpus
end

#memory ⇒ Object (readonly)

Returns the value of attribute memory.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def memory
  @memory
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def name
  @name
end

#pins ⇒ Object (readonly)

Returns the value of attribute pins.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def pins
  @pins
end

#question ⇒ Object (readonly)

Returns the value of attribute question.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def question
  @question
end

#root ⇒ Object (readonly)

Returns the value of attribute root.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def root
  @root
end

#tasks ⇒ Object (readonly)

Returns the value of attribute tasks.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def tasks
  @tasks
end

#timeout_seconds ⇒ Object (readonly)

Returns the value of attribute timeout_seconds.



38
39
40
 
# File 'lib/llm_experiment/experiment.rb', line 38

def timeout_seconds
  @timeout_seconds
end

Class Method Details

.find(start = Dir.pwd) ⇒ Object 



41
42
43
44
45
46
47
48
49
50
 
# File 'lib/llm_experiment/experiment.rb', line 41

def self.find(start = Dir.pwd)
  dir = File.expand_path(start)
  until File.exist?(File.join(dir, "experiment.yml"))
    parent = File.dirname(dir)
    raise ConfigError, "no experiment.yml found in #{start} or any parent" if parent == dir

    dir = parent
  end
  load(dir)
end

.load(dir) ⇒ Object

Raises:



52
53
54
55
56
57
 
# File 'lib/llm_experiment/experiment.rb', line 52

def self.load(dir)
  path = File.join(dir, "experiment.yml")
  raise ConfigError, "missing #{path}" unless File.exist?(path)

  new(Psych.safe_load(File.read(path)) || {}, root: File.expand_path(dir))
end

Instance Method Details

#app(key) ⇒ Object 



79
80
81
 
# File 'lib/llm_experiment/experiment.rb', line 79

def app(key)
  @apps.fetch(key) { raise ConfigError, "unknown app: #{key}" }
end

#prompt_path(task, condition) ⇒ Object 



87
88
89
 
# File 'lib/llm_experiment/experiment.rb', line 87

def prompt_path(task, condition)
  File.join(root, "prompts", task.app, task.id, "#{condition}.txt")
end

#results_dir ⇒ Object 



92
 
# File 'lib/llm_experiment/experiment.rb', line 92

def results_dir = File.join(root, "results")

#results_raw_dir ⇒ Object 



91
 
# File 'lib/llm_experiment/experiment.rb', line 91

def results_raw_dir = File.join(root, "results-raw")

#task(id) ⇒ Object 



83
84
85
 
# File 'lib/llm_experiment/experiment.rb', line 83

def task(id)
  @tasks.find { |t| t.id == id } or raise ConfigError, "unknown task: #{id}"
end

#trial_dir(task_id, agent, condition, stamp) ⇒ Object 



94
95
96
 
# File 'lib/llm_experiment/experiment.rb', line 94

def trial_dir(task_id, agent, condition, stamp)
  File.join(results_raw_dir, task_id, agent, condition, stamp)
end