Class: RubricLLM::Metrics::Faithfulness

Inherits:
Base
  • Object
show all
Defined in:
lib/rubric_llm/metrics/faithfulness.rb

Constant Summary collapse

SYSTEM_PROMPT = 
<<~PROMPT
  You are an evaluation judge. Assess whether the answer is faithful to the provided context.
  A faithful answer only contains information that is supported by the context.

  Respond with JSON only:
  {
    "score": <float 0.0-1.0>,
    "claims": [{"claim": "<statement>", "supported": <true/false>}],
    "reasoning": "<brief explanation>"
  }
PROMPT

Instance Attribute Summary

Attributes inherited from Base

#judge

Instance Method Summary collapse

Methods inherited from Base

#initialize

Constructor Details

This class inherits a constructor from RubricLLM::Metrics::Base

Instance Method Details

#call(question:, answer:, context: []) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/rubric_llm/metrics/faithfulness.rb', line 18

def call(question:, answer:, context: [], **)
  context_chunks = Array(context).map { |chunk| chunk.to_s.strip }.reject(&:empty?)
  return { score: nil, details: { error: "No context provided" } } if context_chunks.empty?

  user_prompt = <<~PROMPT
    Context: #{context_chunks.join("\n\n")}

    Question: #{question}

    Answer: #{answer}

    Evaluate the faithfulness of the answer to the context.
  PROMPT

  result = judge_eval(system_prompt: SYSTEM_PROMPT, user_prompt:)
  normalize(result)
end