Class: Riffer::Evals::ScenarioResult

Inherits:
Object
  • Object
show all
Defined in:
lib/riffer/evals/scenario_result.rb

Overview

Represents the result of evaluating a single scenario.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input:, output:, ground_truth:, results:, messages: []) ⇒ ScenarioResult

– : (input: String, output: String, ground_truth: String?, results: Array, ?messages: Array) -> void



23
24
25
26
27
28
29
 
# File 'lib/riffer/evals/scenario_result.rb', line 23

def initialize(input:, output:, ground_truth:, results:, messages: [])
  @input = input
  @output = output
  @ground_truth = ground_truth
  @results = results
  @messages = messages
end

Instance Attribute Details

#ground_truth ⇒ Object (readonly)

The ground truth used during evaluation.



13
14
15
 
# File 'lib/riffer/evals/scenario_result.rb', line 13

def ground_truth
  @ground_truth
end

#input ⇒ Object (readonly)

The input that was evaluated.



7
8
9
 
# File 'lib/riffer/evals/scenario_result.rb', line 7

def input
  @input
end

#messages ⇒ Object (readonly)

The full message history from the agent conversation.



19
20
21
 
# File 'lib/riffer/evals/scenario_result.rb', line 19

def messages
  @messages
end

#output ⇒ Object (readonly)

The agent output for this scenario.



10
11
12
 
# File 'lib/riffer/evals/scenario_result.rb', line 10

def output
  @output
end

#results ⇒ Object (readonly)

Individual evaluation results.



16
17
18
 
# File 'lib/riffer/evals/scenario_result.rb', line 16

def results
  @results
end

Instance Method Details

#scores ⇒ Object

Returns scores keyed by evaluator class.

– : () -> Hash[singleton(Riffer::Evals::Evaluator), Float]



35
36
37
38
39
40
 
# File 'lib/riffer/evals/scenario_result.rb', line 35

def scores
  acc = {} #: Hash[singleton(Riffer::Evals::Evaluator), Float]
  results.each_with_object(acc) do |result, hash|
    hash[result.evaluator] = result.score
  end
end

#to_h ⇒ Object

Returns a hash representation of the scenario result.

– : () -> Hash[Symbol, untyped]



46
47
48
49
50
51
52
53
54
55
 
# File 'lib/riffer/evals/scenario_result.rb', line 46

def to_h
  {
    input: input,
    output: output,
    ground_truth: ground_truth,
    scores: scores.transform_keys(&:name),
    results: results.map(&:to_h),
    messages: messages.map(&:to_h)
  }
end