Class: Lemans::Result

Inherits:
Object
  • Object
show all
Defined in:
lib/lemans/result.rb

Overview

A single trial result record

Defined Under Namespace

Classes: CostSource, IncompatibleError, Outcome, Phase, Revision, Usage

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(task:, agent:, model:, id: nil, index: nil, profile_digest: nil, task_digest: nil, revision: nil) ⇒ Result

Returns a new instance of Result.



131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/lemans/result.rb', line 131

def initialize(task:, agent:, model:, id: nil, index: nil,
               profile_digest: nil, task_digest: nil, revision: nil)
  @task = task
  @agent = agent
  @model = model
  @index = index
  @profile_digest = profile_digest
  @task_digest = task_digest
  @revision = revision

  @tags = []
  @metadata = {}
  @phases = []

  @id = id || "#{task}__#{SecureRandom.alphanumeric(7)}"
  @outcome = Outcome.new(:pending)
end

Instance Attribute Details

#agent ⇒ Object (readonly)

attributes that must be initialized/specified during construction



121
122
123
 
# File 'lib/lemans/result.rb', line 121

def agent
  @agent
end

#id ⇒ Object (readonly)

attributes that must be initialized/specified during construction



121
122
123
 
# File 'lib/lemans/result.rb', line 121

def id
  @id
end

#index ⇒ Object (readonly)

attributes that must be initialized/specified during construction



121
122
123
 
# File 'lib/lemans/result.rb', line 121

def index
  @index
end

#metadata ⇒ Object

Returns the value of attribute metadata.



124
125
126
 
# File 'lib/lemans/result.rb', line 124

def metadata
  @metadata
end

#model ⇒ Object (readonly)

attributes that must be initialized/specified during construction



121
122
123
 
# File 'lib/lemans/result.rb', line 121

def model
  @model
end

#outcome ⇒ Object (readonly)

outcome-related attributes (we use setter-like methods, not accessors)



129
130
131
 
# File 'lib/lemans/result.rb', line 129

def outcome
  @outcome
end

#phases ⇒ Object (readonly)

Returns the value of attribute phases.



126
127
128
 
# File 'lib/lemans/result.rb', line 126

def phases
  @phases
end

#profile_digest ⇒ Object (readonly)

attributes that must be initialized/specified during construction



121
122
123
 
# File 'lib/lemans/result.rb', line 121

def profile_digest
  @profile_digest
end

#revision ⇒ Object (readonly)

attributes that must be initialized/specified during construction



121
122
123
 
# File 'lib/lemans/result.rb', line 121

def revision
  @revision
end

#reward ⇒ Object (readonly)

outcome-related attributes (we use setter-like methods, not accessors)



129
130
131
 
# File 'lib/lemans/result.rb', line 129

def reward
  @reward
end

#tags ⇒ Object

Returns the value of attribute tags.



124
125
126
 
# File 'lib/lemans/result.rb', line 124

def tags
  @tags
end

#task ⇒ Object (readonly)

attributes that must be initialized/specified during construction



121
122
123
 
# File 'lib/lemans/result.rb', line 121

def task
  @task
end

#task_digest ⇒ Object (readonly)

attributes that must be initialized/specified during construction



121
122
123
 
# File 'lib/lemans/result.rb', line 121

def task_digest
  @task_digest
end

#usage ⇒ Object (readonly)

outcome-related attributes (we use setter-like methods, not accessors)



129
130
131
 
# File 'lib/lemans/result.rb', line 129

def usage
  @usage
end

Class Method Details

.from_json(data) ⇒ Object 



215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
 
# File 'lib/lemans/result.rb', line 215

def from_json(data)
  result = new(
    **data.slice(:task, :agent, :model, :index, :profile_digest, :task_digest),
    # 0.2.x releases spell the id `run:`.
    id: data[:trial] || data[:run],
    revision: revision_from(data)
  )
  result.tags = data[:tags] || []
  result.metadata = data[:metadata] || {}
  phases_from(data).each { result.phases << it }
  if data[:outcome]
    result.completed!(
      Outcome.from_json(data[:outcome]),
      data[:usage] && Usage.from_json(data[:usage]),
      # The recorded duration wins over the one derived from phases:
      # older writers timed a slightly wider span.
      duration: data[:duration] || data[:duration_sec]
    )
  end
  result.graded!(data[:reward]) unless data[:reward].nil?
  result
end

.from_task(definition) ⇒ Object 



238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/lemans/result.rb', line 238

def from_task(definition, **)
  config = definition.config
  result = new(
    task: definition.name,
    agent: config.agent_name,
    model: config.models.first,
    profile_digest: config.digest,
    task_digest: definition.digest,
    revision: config.revision,
    **
  )
  result.tags = definition.tags
  result.metadata = definition.metadata
  result
end

Instance Method Details

#as_json ⇒ Object 



202
203
204
205
206
207
208
209
210
211
212
 
# File 'lib/lemans/result.rb', line 202

def as_json(**)
  {
    trial: id, task:, agent:, model:, index:,
    profile_digest:, task_digest:, revision: revision&.as_json,
    lemans_version: VERSION,
    tags:, metadata:, phases: phases.map(&:as_json),
    reward:, outcome: outcome.as_json, usage: usage&.as_json, duration:,
    started_at: started_at&.iso8601,
    finished_at: finished_at&.iso8601
  }.compact
end

#completed!(outcome, usage = nil, duration: nil) ⇒ Object 



180
181
182
183
184
185
 
# File 'lib/lemans/result.rb', line 180

def completed!(outcome, usage = nil, duration: nil)
  @outcome = outcome.is_a?(Outcome) ? outcome : Outcome.new(outcome)
  @usage = usage
  @duration = duration if duration
  self
end

#detail ⇒ Object 



168
 
# File 'lib/lemans/result.rb', line 168

def detail = outcome.detail

#duration ⇒ Object 



176
177
178
 
# File 'lib/lemans/result.rb', line 176

def duration
  @duration || (finished_at && started_at && (finished_at - started_at).round(1))
end

#failed!(reason, detail) ⇒ Object 



192
193
194
195
196
197
198
199
200
 
# File 'lib/lemans/result.rb', line 192

def failed!(reason, detail)
  # do not override already stored error
  # (in case we have rescue cascades)
  return self if outcome.invalid? && !outcome.pending?

  @outcome = Outcome.new(reason, detail)
  @reward = nil
  self
end

#finished_at ⇒ Object 



164
 
# File 'lib/lemans/result.rb', line 164

def finished_at = phases.last&.finished_at

#graded!(reward) ⇒ Object 



187
188
189
190
 
# File 'lib/lemans/result.rb', line 187

def graded!(reward)
  @reward = reward
  self
end

#invalid? ⇒ Boolean

Returns:

  • (Boolean) 


174
 
# File 'lib/lemans/result.rb', line 174

def invalid? = outcome.invalid?

#pending? ⇒ Boolean

Returns:

  • (Boolean) 


170
 
# File 'lib/lemans/result.rb', line 170

def pending? = outcome.pending?

#phase_finished(name, finished_at = nil) ⇒ Object

Raises:

  • (ArgumentError) 


155
156
157
158
159
160
 
# File 'lib/lemans/result.rb', line 155

def phase_finished(name, finished_at = nil)
  raise ArgumentError, "no current phase" if phases.empty?
  raise ArgumentError, "can not finish phase #{name}: current one is #{phases.last.name}" if phases.last.name != name

  phases.last.finish!(finished_at)
end

#phase_started(name, started_at = nil) ⇒ Object

Raises:

  • (ArgumentError) 


149
150
151
152
153
 
# File 'lib/lemans/result.rb', line 149

def phase_started(name, started_at = nil)
  raise ArgumentError, "previous phase hasn't finished yet" if phases.last && !phases.last.finished?

  phases << Phase.new(name, started_at:)
end

#scored? ⇒ Boolean

Returns:

  • (Boolean) 


172
 
# File 'lib/lemans/result.rb', line 172

def scored? = outcome.scored?

#started_at ⇒ Object 



162
 
# File 'lib/lemans/result.rb', line 162

def started_at = phases.first&.started_at

#status ⇒ Object 



166
 
# File 'lib/lemans/result.rb', line 166

def status = outcome.status