Class: RubricLLM::Metrics::FactualAccuracy

Inherits:
Base
  • Object
show all
Defined in:
lib/rubric_llm/metrics/factual_accuracy.rb

Constant Summary collapse

SYSTEM_PROMPT = 
<<~PROMPT
  You are an evaluation judge. Compare the factual claims in the candidate answer against the reference answer.
  Identify any discrepancies where the candidate states something different from the reference.

  Respond with JSON only:
  {
    "score": <float 0.0-1.0>,
    "discrepancies": [{"claim": "<candidate claim>", "reference": "<what reference says>", "severity": "minor|major"}],
    "reasoning": "<brief explanation>"
  }
PROMPT

Instance Attribute Summary

Attributes inherited from Base

#judge

Instance Method Summary collapse

Methods inherited from Base

#initialize

Constructor Details

This class inherits a constructor from RubricLLM::Metrics::Base

Instance Method Details

#call(answer:, ground_truth: nil) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/rubric_llm/metrics/factual_accuracy.rb', line 18

def call(answer:, ground_truth: nil, **)
  return { score: nil, details: { error: "No ground truth provided" } } if ground_truth.nil?

  user_prompt = <<~PROMPT
    Candidate Answer: #{answer}

    Reference Answer: #{ground_truth}

    Compare the factual claims and identify any discrepancies.
  PROMPT

  result = judge_eval(system_prompt: SYSTEM_PROMPT, user_prompt:)
  normalize(result)
end