14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
# File 'lib/legion/llm/inference/steps/rag_guard.rb', line 14
def check_rag_faithfulness
context = @enrichments.dig('rag:context_retrieval', :data, :entries)
unless context&.any?
log_step_debug(:rag_guard, :skipped, reason: :no_context)
return
end
unless defined?(Hooks::RagGuard)
log.warn('[rag_guard] RAG context present but no Hooks::RagGuard registered — faithfulness check skipped')
log_step_debug(:rag_guard, :skipped, reason: :hook_missing, context_count: context.size)
return
end
response_text = @raw_response.respond_to?(:content) ? @raw_response.content : @raw_response.to_s
log_step_debug(:rag_guard, :checking, context_count: context.size, response_chars: response_text.length)
result = Hooks::RagGuard.check_rag_faithfulness(
response: response_text,
context: context.map { |e| e[:content] }.join("\n"),
threshold: rag_faithfulness_threshold
)
unless result.is_a?(Hash)
log_step_debug(:rag_guard, :skipped, reason: :unexpected_result)
return
end
if result[:faithful] == true
record_rag_faithfulness_audit(:success, result)
log_step_debug(:rag_guard, :passed)
return
end
detail = result[:details] || result[:reason] || 'faithfulness check failed'
log.warn("[rag_guard] RAG faithfulness failure: #{detail}")
log_step_info(:rag_guard, :blocked, reason_chars: detail.to_s.length)
@warnings << { type: :rag_faithfulness_failure, message: detail, result: result }
record_rag_faithfulness_audit(:failure, result, detail: detail)
@timeline.record(
category: :quality, key: 'rag:faithfulness_failure',
direction: :internal, detail: detail,
from: 'rag_guard', to: 'pipeline'
)
return unless rag_block_on_failure?
raise Legion::LLM::PipelineError.new("RAG faithfulness failed: #{detail}", step: :rag_guard)
rescue Legion::LLM::PipelineError
raise
rescue StandardError => e
@warnings << "RagGuard error: #{e.message}"
handle_exception(e, level: :warn, operation: 'llm.pipeline.steps.rag_guard')
end
|