38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
|
# File 'lib/legion/guardrails.rb', line 38
def check(question:, context:, answer:, threshold: 3)
return { relevant: true, reason: 'no LLM' } unless defined?(Legion::LLM)
result = Legion::LLM.chat(
message: [
{ role: 'system',
content: 'Rate 1-5 how relevant the answer is to the question given the context. Reply ONLY with the number.' },
{ role: 'user', content: "Question: #{question}\nContext: #{context}\nAnswer: #{answer}" }
],
caller: Guardrails::SYSTEM_CALLER
)
score = result[:content].to_s.strip.to_i
relevant = score >= threshold
Legion::Logging.warn "[Guardrails] RAGRelevancy rejected answer: score=#{score} threshold=#{threshold}" if !relevant && defined?(Legion::Logging)
{ relevant: relevant, score: score, threshold: threshold }
rescue StandardError => e
Legion::Logging.warn "Guardrails::RAGRelevancy#check failed: #{e.message}" if defined?(Legion::Logging)
{ relevant: true, reason: 'check failed' }
end
|