Class: Phronomy::Testing::Eval::Scorer::LlmJudge

Inherits:
Base
  • Object
show all
Defined in:
lib/phronomy/testing/eval/scorer/llm_judge.rb

Constant Summary collapse

DEFAULT_PROMPT = 
<<~PROMPT
  You are an impartial judge evaluating the quality of an AI assistant response.
  Rate the response on a scale from 0.0 (completely wrong or unhelpful) to 1.0 (perfect).
  Respond with ONLY a single decimal number between 0.0 and 1.0 — no other text.

  Question: %<input>s
  Expected answer: %<expected>s
  Actual response: %<actual>s

  Score:
PROMPT

Instance Method Summary collapse

Constructor Details

#initialize(model:, prompt_template: DEFAULT_PROMPT, raise_on_error: false) ⇒ LlmJudge

Returns a new instance of LlmJudge.



20
21
22
23
24
 
# File 'lib/phronomy/testing/eval/scorer/llm_judge.rb', line 20

def initialize(model:, prompt_template: DEFAULT_PROMPT, raise_on_error: false)
  @model = model
  @prompt_template = prompt_template
  @raise_on_error = raise_on_error
end

Instance Method Details

#score(actual:, expected:, input: nil) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/phronomy/testing/eval/scorer/llm_judge.rb', line 26

def score(actual:, expected:, input: nil)
  prompt = format(
    @prompt_template,
    input: input.to_s,
    expected: expected.to_s,
    actual: actual.to_s
  )
  response = Phronomy::Runtime.instance.offload.submit do
    RubyLLM.chat(model: @model).ask(prompt)
  end.blocking_wait
  response.content.to_s.strip.scan(/-?\d+\.?\d*/).first.to_f.clamp(0.0, 1.0)
rescue => error
  raise if @raise_on_error
  warn "[LlmJudge] Scoring failed: #{error.message}"
  0.0
end