Module: Legion::LLM::QualityChecker

Extended by:
Legion::Logging::Helper
Defined in:
lib/legion/llm/quality_checker.rb

Defined Under Namespace

Classes: QualityResult

Constant Summary collapse

REPETITION_MIN_LENGTH = 
20
REPETITION_THRESHOLD = 
3
DEFAULT_QUALITY_THRESHOLD = 
50
REFUSAL_PATTERNS = 
[
  /\bI (?:can(?:'t|not)|cannot|won't|am unable to)\b/i,
  /\bI'm not able to\b/i,
  /\bas an AI\b/i,
  /\bI don't have (?:the ability|access)\b/i
].freeze

Class Method Summary collapse

Class Method Details

.check(response, quality_threshold: DEFAULT_QUALITY_THRESHOLD, json_expected: false, quality_check: nil) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/legion/llm/quality_checker.rb', line 25

def check(response, quality_threshold: DEFAULT_QUALITY_THRESHOLD, json_expected: false, quality_check: nil)
  failures = []
  content = response.content

  failures << :empty_response if content.nil? || content.strip.empty?

  unless failures.include?(:empty_response)
    failures << :too_short if content.length < quality_threshold
    failures << :repetition if repetitive?(content)
    failures << :truncated if truncated?(content)
    failures << :refusal if refusal?(content)
    failures << :json_parse_failure if json_expected && !valid_json?(content)
  end

  failures << :custom_check_failed if quality_check.respond_to?(:call) && !quality_check.call(response)

  QualityResult.new(passed: failures.empty?, failures: failures)
end