Class: Lemans::Task

Inherits:
Object
  • Object
show all
Defined in:
lib/lemans/task.rb

Overview

One task: an instruction, an environment, a verifier, a solution. Anything lemans does not understand belongs under metadata, copied untouched.

Defined Under Namespace

Classes: ImageSpec

Constant Summary collapse

INSTRUCTION = 
"instruction.md"
ENVIRONMENT_DIR = 
"environment"
TESTS_DIR = 
"tests"
SOLUTION_DIR = 
"solution"
FLAT_TEST = 
"verification_test.rb"
FLAT_SOLUTION = 
"solution.patch"
FLAT_SEED = 
"environment.patch"
FRONTMATTER = 
/\A---\n(.*?)\n---\n/m

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config, dir:, bench:) ⇒ Task

Returns a new instance of Task.



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/lemans/task.rb', line 85

def initialize(config, dir:, bench:)
  @dir = Pathname(dir)
  @bench = bench
  @name = config["name"] || @dir.basename.to_s
  @description = config["description"]
  @difficulty = config["difficulty"]
  @tags = Array(config["tags"]).freeze
  @metadata = (config["metadata"] || {}).freeze
  @files = SetupFiles.call(config["files"], root: @dir, label: @dir)
  @restore = config.key?("restore") ? RestorePaths.call(config["restore"], label: "#{@dir}: restore") : nil

  validate!(config)
  # Recorded on every result: without it a reward cannot say which task version it measured.
  @digest = TreeDigest.call(@dir)[0, 16]
  freeze
end

Instance Attribute Details

#bench ⇒ Object (readonly)

Returns the value of attribute bench.



56
57
58
 
# File 'lib/lemans/task.rb', line 56

def bench
  @bench
end

#description ⇒ Object (readonly)

Returns the value of attribute description.



56
57
58
 
# File 'lib/lemans/task.rb', line 56

def description
  @description
end

#difficulty ⇒ Object (readonly)

Returns the value of attribute difficulty.



56
57
58
 
# File 'lib/lemans/task.rb', line 56

def difficulty
  @difficulty
end

#digest ⇒ Object (readonly)

Returns the value of attribute digest.



56
57
58
 
# File 'lib/lemans/task.rb', line 56

def digest
  @digest
end

#dir ⇒ Object (readonly)

Returns the value of attribute dir.



56
57
58
 
# File 'lib/lemans/task.rb', line 56

def dir
  @dir
end

#metadata ⇒ Object (readonly)

Returns the value of attribute metadata.



56
57
58
 
# File 'lib/lemans/task.rb', line 56

def metadata
  @metadata
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



56
57
58
 
# File 'lib/lemans/task.rb', line 56

def name
  @name
end

#tags ⇒ Object (readonly)

Returns the value of attribute tags.



56
57
58
 
# File 'lib/lemans/task.rb', line 56

def tags
  @tags
end

Class Method Details

.frontmatter(dir) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/lemans/task.rb', line 63

def self.frontmatter(dir)
  content = dir.join(INSTRUCTION).read
  match = content.match(FRONTMATTER)
  unless match
    # Opens like frontmatter but never matches: silently dropping every
    # declared key (and leaking the raw block to the agent) is worse than
    # refusing. CRLF endings and a missing final newline are the usual causes.
    raise ConfigError, "#{dir.join(INSTRUCTION)}: frontmatter opens with --- but never closes" if
      content.start_with?("---")

    return {}
  end
  config = YAML.safe_load(match[1], aliases: true) || {}
  raise ConfigError, "#{dir.join(INSTRUCTION)}: frontmatter must be a mapping" unless config.is_a?(Hash)

  config
rescue Errno::ENOENT
  {} # fallback to validate!
rescue Psych::Exception => e
  raise ConfigError, "#{dir.join(INSTRUCTION)}: #{e.message}"
end

.load(dir, bench:) ⇒ Object 



58
59
60
61
 
# File 'lib/lemans/task.rb', line 58

def self.load(dir, bench:)
  dir = Pathname(dir)
  new(frontmatter(dir), dir: dir, bench: bench)
end

Instance Method Details

#environment_context ⇒ Object 



107
 
# File 'lib/lemans/task.rb', line 107

def environment_context = dir.join(ENVIRONMENT_DIR)

#environment_dockerfile ⇒ Object 



109
 
# File 'lib/lemans/task.rb', line 109

def environment_dockerfile = environment_context.join("Dockerfile")

#environment_image ⇒ Object 



131
132
133
134
135
136
137
 
# File 'lib/lemans/task.rb', line 131

def environment_image
  if bench.environment.image
    ImageSpec.registry(bench.environment.image)
  else
    ImageSpec.dockerfile(environment_dockerfile, slug: name)
  end
end

#instruction ⇒ Object

The story alone: frontmatter is for the harness, never for the agent.



103
 
# File 'lib/lemans/task.rb', line 103

def instruction = instruction_path.read.sub(FRONTMATTER, "")

#instruction_path ⇒ Object 



105
 
# File 'lib/lemans/task.rb', line 105

def instruction_path = dir.join(INSTRUCTION)

#restore_paths ⇒ Object 



151
 
# File 'lib/lemans/task.rb', line 151

def restore_paths = @restore || bench.verifier.restore_paths

#setup_files(phase) ⇒ Object 



139
140
141
142
143
144
145
 
# File 'lib/lemans/task.rb', line 139

def setup_files(phase)
  declared = @files.fetch(phase.to_sym, [])
  seed = Pathname(FLAT_SEED)
  return declared unless phase.to_sym == :environment && dir.join(seed).file? && !declared.include?(seed)

  declared + [seed]
end

#solution? ⇒ Boolean

Returns:

  • (Boolean) 


149
 
# File 'lib/lemans/task.rb', line 149

def solution? = solution_files.any?

#solution_context ⇒ Object 



147
 
# File 'lib/lemans/task.rb', line 147

def solution_context = dir.join(SOLUTION_DIR)

#solution_files ⇒ Object 



123
124
125
126
127
128
129
 
# File 'lib/lemans/task.rb', line 123

def solution_files
  if solution_context.directory?
    expand(solution_context)
  else
    flat(FLAT_SOLUTION)
  end
end

#test_files ⇒ Object

[absolute, remote-relative] pairs.



115
116
117
118
119
120
121
 
# File 'lib/lemans/task.rb', line 115

def test_files
  if tests_dir.directory?
    expand(tests_dir)
  else
    flat(FLAT_TEST)
  end
end

#tests_dir ⇒ Object

Stays on the harness side while the agent works; uploaded into the sandbox only at verification.



112
 
# File 'lib/lemans/task.rb', line 112

def tests_dir = dir.join(TESTS_DIR)

#to_h ⇒ Object 



153
154
155
156
157
158
159
160
161
 
# File 'lib/lemans/task.rb', line 153

def to_h
  {
    name: name,
    description: description,
    difficulty: difficulty,
    tags: tags,
    metadata: metadata
  }.compact
end