Class: OmniAgent::Eval::Judge

Inherits:
Object
  • Object
show all
Defined in:
lib/omni_agent/eval/judge.rb

Constant Summary collapse

GRADING_PROMPT = 
<<~PROMPT
  You are grading an AI agent's output against a single criterion.

  Output: %<output>s

  Criterion: %<criteria>s

  Respond with strict JSON only, no other text: {"score": <float between 0 and 1>, "reason": "<short explanation>"}
PROMPT

Instance Method Summary collapse

Constructor Details

#initialize(provider: nil, model: nil) ⇒ Judge

Returns a new instance of Judge.



16
17
18
19
 
# File 'lib/omni_agent/eval/judge.rb', line 16

def initialize(provider: nil, model: nil)
  @provider = provider
  @model = model
end

Instance Method Details

#call(criteria:, output:, fallback_provider: nil) ⇒ Object 



21
22
23
24
25
26
27
 
# File 'lib/omni_agent/eval/judge.rb', line 21

def call(criteria:, output:, fallback_provider: nil)
  judge_provider = resolve_provider(fallback_provider)
  prompt = format(GRADING_PROMPT, output: output, criteria: criteria)

  response = judge_provider.chat(messages: [ { role: "user", content: prompt } ])
  parse_score(response.content)
end