Class: RubricLLM::Metrics::Correctness

Inherits:
Base
  • Object
show all
Defined in:
lib/rubric_llm/metrics/correctness.rb

Constant Summary collapse

SYSTEM_PROMPT = 
<<~PROMPT
  You are an evaluation judge. Assess whether the answer matches the ground truth.
  Consider semantic equivalence, not just exact string matching.

  Respond with JSON only:
  {
    "score": <float 0.0-1.0>,
    "reasoning": "<brief explanation>"
  }
PROMPT

Instance Attribute Summary

Attributes inherited from Base

#judge

Instance Method Summary collapse

Methods inherited from Base

#initialize

Constructor Details

This class inherits a constructor from RubricLLM::Metrics::Base

Instance Method Details

#call(question:, answer:, ground_truth: nil) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/rubric_llm/metrics/correctness.rb', line 17

def call(question:, answer:, ground_truth: nil, **)
  return { score: nil, details: { error: "No ground truth provided" } } if ground_truth.nil?

  user_prompt = <<~PROMPT
    Question: #{question}

    Answer: #{answer}

    Ground Truth: #{ground_truth}

    Evaluate the correctness of the answer compared to the ground truth.
  PROMPT

  result = judge_eval(system_prompt: SYSTEM_PROMPT, user_prompt:)
  normalize(result)
end