Module: Legion::LLM::Hooks::RagGuard

Extended by:
Legion::Logging::Helper
Defined in:
lib/legion/llm/hooks/rag_guard.rb

Class Method Summary collapse

Class Method Details

.check_rag_faithfulness(response:, context:, threshold: nil, evaluators: nil) ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/legion/llm/hooks/rag_guard.rb', line 11

def check_rag_faithfulness(response:, context:, threshold: nil, evaluators: nil, **)
  return { faithful: true, reason: :eval_unavailable } unless eval_available?

  resolved_threshold = threshold || settings_threshold
  resolved_evaluators = evaluators || settings_evaluators

  scores = {}
  flagged = []

  resolved_evaluators.each do |evaluator_name|
    score = run_evaluator(evaluator_name, response: response, context: context)
    scores[evaluator_name] = score
    flagged << evaluator_name if score < resolved_threshold
  end

  faithful = flagged.empty?
  details = build_details(scores, resolved_threshold, faithful)

  { faithful: faithful, scores: scores, flagged_evaluators: flagged, details: details }
rescue StandardError => e
  handle_exception(e, level: :warn)
  { faithful: true, reason: :eval_error }
end