Class: RubyLLM::Contract::Eval::CaseExecutor

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/contract/eval/case_executor.rb

Instance Method Summary collapse

Constructor Details

#initialize(step:, scorer: nil, normalizer: StepResultNormalizer.new, result_builder: CaseResultBuilder.new, step_expectation_applier: StepExpectationApplier.new) ⇒ CaseExecutor

Returns a new instance of CaseExecutor.



7
8
9
10
11
12
13
14
15
 
# File 'lib/ruby_llm/contract/eval/case_executor.rb', line 7

def initialize(step:, scorer: nil, normalizer: StepResultNormalizer.new,
               result_builder: CaseResultBuilder.new,
               step_expectation_applier: StepExpectationApplier.new)
  @step = step
  @scorer = scorer || CaseScorer.new(step: step)
  @normalizer = normalizer
  @result_builder = result_builder
  @step_expectation_applier = step_expectation_applier
end

Instance Method Details

#call(test_case:, context:) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/ruby_llm/contract/eval/case_executor.rb', line 17

def call(test_case:, context:)
  run_result = @step.run(test_case.input, context: context)
  step_result = @normalizer.call(run_result)
  evaluation = @scorer.call(test_case: test_case, step_result: step_result)
  result = @result_builder.call(test_case: test_case, step_result: step_result, evaluation: evaluation)

  @step_expectation_applier.call(result: result, run_result: run_result, test_case: test_case)
rescue RubyLLM::Contract::Error => error
  raise unless missing_adapter?(error)

  skipped_result(test_case, error.message)
end