Class: Riffer::Evals::ScenarioResult

Inherits:
Object
  • Object
show all
Defined in:
lib/riffer/evals/scenario_result.rb,
sig/generated/riffer/evals/scenario_result.rbs

Overview

Represents the result of evaluating a single scenario.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input:, output:, ground_truth:, results:, messages: [], token_usage: nil) ⇒ ScenarioResult

-- : (input: String, output: String, ground_truth: String?, results: Array, ?messages: Array, ?token_usage: Riffer::Providers::TokenUsage?) -> void

Parameters:



26
27
28
29
30
31
32
33
 
# File 'lib/riffer/evals/scenario_result.rb', line 26

def initialize(input:, output:, ground_truth:, results:, messages: [], token_usage: nil)
  @input = input
  @output = output
  @ground_truth = ground_truth
  @results = results
  @messages = messages
  @token_usage = token_usage
end

Instance Attribute Details

#ground_truth ⇒ String? (readonly)

The ground truth used during evaluation.

Returns:

  • (String, nil) 


13
14
15
 
# File 'lib/riffer/evals/scenario_result.rb', line 13

def ground_truth
  @ground_truth
end

#input ⇒ String (readonly)

The input that was evaluated.

Returns:

  • (String) 


7
8
9
 
# File 'lib/riffer/evals/scenario_result.rb', line 7

def input
  @input
end

#messages ⇒ Array[Riffer::Messages::Base] (readonly)

The full message history from the agent conversation.

Returns:



19
20
21
 
# File 'lib/riffer/evals/scenario_result.rb', line 19

def messages
  @messages
end

#output ⇒ String (readonly)

The agent output for this scenario.

Returns:

  • (String) 


10
11
12
 
# File 'lib/riffer/evals/scenario_result.rb', line 10

def output
  @output
end

#results ⇒ Array[Riffer::Evals::Result] (readonly)

Individual evaluation results.

Returns:



16
17
18
 
# File 'lib/riffer/evals/scenario_result.rb', line 16

def results
  @results
end

#token_usage ⇒ Riffer::Providers::TokenUsage? (readonly)

Token usage the agent under test spent generating this scenario's output.

Returns:



22
23
24
 
# File 'lib/riffer/evals/scenario_result.rb', line 22

def token_usage
  @token_usage
end

Instance Method Details

#evaluator_token_usage ⇒ Riffer::Providers::TokenUsage?

Returns the summed token usage across this scenario's LLM-as-judge evaluators, or nil when none reported usage.

-- : () -> Riffer::Providers::TokenUsage?

Returns:



51
52
53
 
# File 'lib/riffer/evals/scenario_result.rb', line 51

def evaluator_token_usage
  results.map(&:token_usage).compact.reduce(:+)
end

#scores ⇒ Hash[singleton(Riffer::Evals::Evaluator), Float]

Returns scores keyed by evaluator class.

-- : () -> Hash[singleton(Riffer::Evals::Evaluator), Float]

Returns:



39
40
41
42
43
44
 
# File 'lib/riffer/evals/scenario_result.rb', line 39

def scores
  acc = {} #: Hash[singleton(Riffer::Evals::Evaluator), Float]
  results.each_with_object(acc) do |result, hash|
    hash[result.evaluator] = result.score
  end
end

#to_h ⇒ Hash[Symbol, untyped]

Returns a hash representation of the scenario result.

-- : () -> Hash[Symbol, untyped]

Returns:

  • (Hash[Symbol, untyped]) 


59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/riffer/evals/scenario_result.rb', line 59

def to_h
  {
    input: input,
    output: output,
    ground_truth: ground_truth,
    scores: scores.transform_keys(&:name),
    results: results.map(&:to_h),
    messages: messages.map(&:to_h),
    token_usage: token_usage&.to_h,
    evaluator_token_usage: evaluator_token_usage&.to_h
  }
end