Class: OmniAgent::Eval::JudgeAssertion

Inherits:
Object
  • Object
show all
Defined in:
lib/omni_agent/eval/judge_assertion.rb

Instance Method Summary collapse

Constructor Details

#initialize(criteria, threshold: 0.7, provider: nil, model: nil) ⇒ JudgeAssertion

Returns a new instance of JudgeAssertion.



4
5
6
7
8
 
# File 'lib/omni_agent/eval/judge_assertion.rb', line 4

def initialize(criteria, threshold: 0.7, provider: nil, model: nil)
  @criteria = criteria
  @threshold = threshold
  @judge = Judge.new(provider: provider, model: model)
end

Instance Method Details

#call(run) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
 
# File 'lib/omni_agent/eval/judge_assertion.rb', line 10

def call(run)
  verdict = @judge.call(criteria: @criteria, output: run.output, fallback_provider: run.agent.provider)
  passed = verdict[:score] >= @threshold

  Outcome.new(
    passed: passed,
    message: "judge score #{verdict[:score]} (threshold #{@threshold}) - #{verdict[:reason]}",
    score: verdict[:score]
  )
end