Class: Riffer::Evals::Runner

Inherits:
Object
  • Object
show all
Defined in:
lib/riffer/evals/runner.rb

Overview

Orchestrates running multiple evaluators against agent output.

The Runner takes a set of metrics and runs each evaluator, collecting results into a RunResult.

runner = Riffer::Evals::Runner.new(metrics: [metric1, metric2])
run_result = runner.run(
  input: "What is the capital of France?",
  output: "The capital of France is Paris.",
  context: {}
)

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(metrics:) ⇒ Runner

Initializes a new runner.

: (metrics: Array) -> void



23
24
25
 
# File 'lib/riffer/evals/runner.rb', line 23

def initialize(metrics:)
  @metrics = metrics
end

Instance Attribute Details

#metrics ⇒ Object (readonly)

The metrics to evaluate.



18
19
20
 
# File 'lib/riffer/evals/runner.rb', line 18

def metrics
  @metrics
end

Instance Method Details

#run(input:, output:, context: nil) ⇒ Object

Runs all evaluators and collects results.

: (input: String, output: String, ?context: Hash[Symbol, untyped]?) -> Riffer::Evals::RunResult



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/riffer/evals/runner.rb', line 30

def run(input:, output:, context: nil)
  results = metrics.map do |metric|
    evaluator_class = metric.evaluator_class
    raise Riffer::ArgumentError, "Evaluator not found: #{metric.evaluator_identifier}" unless evaluator_class

    evaluator = evaluator_class.new
    evaluator.evaluate(input: input, output: output, context: context)
  end

  Riffer::Evals::RunResult.new(
    input: input,
    output: output,
    context: context,
    results: results,
    metrics: metrics
  )
end