Class: RubyLLM::Contract::Eval::Runner

Inherits:
Object
  • Object
show all
Includes:
Concerns::ContextHelpers, ContractDetailBuilder, TraitEvaluator
Defined in:
lib/ruby_llm/contract/eval/runner.rb

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(step:, dataset:, context: {}, concurrency: nil) ⇒ Runner

Returns a new instance of Runner.



15
16
17
18
19
20
 
# File 'lib/ruby_llm/contract/eval/runner.rb', line 15

def initialize(step:, dataset:, context: {}, concurrency: nil)
  @step = step
  @dataset = dataset
  @context = context
  @concurrency = concurrency
end

Class Method Details

.run(step:, dataset:, context: {}, concurrency: nil) ⇒ Object 



11
12
13
 
# File 'lib/ruby_llm/contract/eval/runner.rb', line 11

def self.run(step:, dataset:, context: {}, concurrency: nil)
  new(step: step, dataset: dataset, context: context, concurrency: concurrency).run
end

Instance Method Details

#run ⇒ Object 



22
23
24
25
26
27
28
29
30
 
# File 'lib/ruby_llm/contract/eval/runner.rb', line 22

def run
  results = if @concurrency && @concurrency > 1
              run_concurrent
            else
              @dataset.cases.map { |test_case| evaluate_case(test_case) }
            end
  step_name = @step.respond_to?(:name) ? @step.name : @step.to_s
  Report.new(dataset_name: @dataset.name, results: results, step_name: step_name)
end