26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
# File 'lib/legion/llm/quality/checker.rb', line 26
def check(response, quality_threshold: DEFAULT_QUALITY_THRESHOLD, json_expected: false, quality_check: nil)
failures = []
content = response.content
has_tool_calls = response.respond_to?(:tool_calls) && response.tool_calls&.any?
failures << :empty_response if !has_tool_calls && (content.nil? || content.strip.empty?)
unless failures.include?(:empty_response)
if quality_setting(:too_short, false) && content.length < quality_threshold
failures << :too_short
log.debug "[llm][quality] check=too_short result=fail content_length=#{content.length} threshold=#{quality_threshold}"
end
if repetitive?(content)
failures << :repetition
log.debug "[llm][quality] check=repetition result=fail content_length=#{content.length}"
end
if quality_setting(:truncated, false) && truncated?(content)
failures << :truncated
log.debug '[llm][quality] check=truncated result=fail'
end
if quality_setting(:refusal, false) && refusal?(content)
failures << :refusal
log.debug '[llm][quality] check=refusal result=fail'
end
if json_expected && !valid_json?(content)
failures << :json_parse_failure
log.debug '[llm][quality] check=json_parse result=fail'
end
end
failures << :custom_check_failed if quality_check.respond_to?(:call) && !quality_check.call(response)
result = QualityResult.new(passed: failures.empty?, failures: failures)
if result.passed
log.info "[llm][quality] action=check result=passed content_length=#{content.to_s.length}"
else
log.warn "[llm][quality] action=check result=failed failures=#{failures.join(',')}"
end
result
end
|