Module: Ask::Eval::Assertions::Deterministic

Defined in:
lib/ask/eval/assertions/deterministic.rb

Overview

Deterministic (non-LLM) assertion checks. Each method returns { passed: Boolean, score: Float, reason: String }.

Class Method Summary collapse

Class Method Details

.contains(output, value:) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

  • value (String) —

    substring to find

Returns:

  • (Hash) —

    { passed:, score:, reason: }



16
17
18
19
20
21
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 16

def contains(output, value:)
  passed = output.to_s.include?(value.to_s)
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output contains #{value.inspect}" :
                          "Output does not contain #{value.inspect}")
end

.email(output, **_kwargs) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

Returns:

  • (Hash) —

    { passed:, score:, reason: }



131
132
133
134
135
136
137
138
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 131

def email(output, **_kwargs)
  # Simple but effective email regex
  pattern = /\A[^@\s]+@[^@\s]+\.[^@\s]+\z/
  passed = pattern.match?(output.to_s.strip)
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output is a valid email address" :
                          "Output is not a valid email address")
end

.ends_with(output, suffix:) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

  • suffix (String) —

    expected suffix

Returns:

  • (Hash) —

    { passed:, score:, reason: }



78
79
80
81
82
83
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 78

def ends_with(output, suffix:)
  passed = output.to_s.end_with?(suffix.to_s)
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output ends with #{suffix.inspect}" :
                          "Output does not end with #{suffix.inspect}")
end

.equals(output, value:) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

  • value (String) —

    exact expected value

Returns:

  • (Hash) —

    { passed:, score:, reason: }



88
89
90
91
92
93
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 88

def equals(output, value:)
  passed = output.to_s == value.to_s
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output equals #{value.inspect}" :
                          "Output does not equal #{value.inspect}")
end

.is_json(output, **_kwargs) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

Returns:

  • (Hash) —

    { passed:, score:, reason: }



47
48
49
50
51
52
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 47

def is_json(output, **_kwargs)
  JSON.parse(output.to_s)
  result(true, score: 1.0, reason: "Output is valid JSON")
rescue JSON::ParserError => e
  result(false, score: 0.0, reason: "Output is not valid JSON: #{e.message}")
end

.max_length(output, max:) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

  • max (Integer) —

    maximum string length

Returns:

  • (Hash) —

    { passed:, score:, reason: }



109
110
111
112
113
114
115
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 109

def max_length(output, max:)
  len = output.to_s.length
  passed = len <= max
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output length #{len} <= #{max}" :
                          "Output length #{len} > #{max}")
end

.max_tokens(output, max:) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

  • max (Integer) —

    maximum allowed tokens

Returns:

  • (Hash) —

    { passed:, score:, reason: }



57
58
59
60
61
62
63
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 57

def max_tokens(output, max:)
  count = approximate_tokens(output.to_s)
  passed = count <= max
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output has ~#{count} tokens (max: #{max})" :
                          "Output has ~#{count} tokens (max: #{max})")
end

.min_length(output, min:) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

  • min (Integer) —

    minimum string length

Returns:

  • (Hash) —

    { passed:, score:, reason: }



98
99
100
101
102
103
104
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 98

def min_length(output, min:)
  len = output.to_s.length
  passed = len >= min
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output length #{len} >= #{min}" :
                          "Output length #{len} < #{min}")
end

.not_contains(output, value:) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

  • value (String) —

    substring that should be absent

Returns:

  • (Hash) —

    { passed:, score:, reason: }



26
27
28
29
30
31
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 26

def not_contains(output, value:)
  passed = !output.to_s.include?(value.to_s)
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output does not contain #{value.inspect}" :
                          "Output contains #{value.inspect}")
end

.regex(output, pattern:) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

  • pattern (Regexp, String) —

    regex pattern to match

Returns:

  • (Hash) —

    { passed:, score:, reason: }



36
37
38
39
40
41
42
43
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 36

def regex(output, pattern:)
  re = pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern.to_s)
  match = re.match(output.to_s)
  passed = !match.nil?
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output matches #{re.inspect}#{" at #{match[0].inspect}" if match}" :
                          "Output does not match #{re.inspect}")
end

.starts_with(output, prefix:) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

  • prefix (String) —

    expected prefix

Returns:

  • (Hash) —

    { passed:, score:, reason: }



68
69
70
71
72
73
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 68

def starts_with(output, prefix:)
  passed = output.to_s.start_with?(prefix.to_s)
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output starts with #{prefix.inspect}" :
                          "Output does not start with #{prefix.inspect}")
end

.url(output, **_kwargs) ⇒ Hash

Returns { passed:, score:, reason: }.

Parameters:

  • output (String) —

    the output to check

Returns:

  • (Hash) —

    { passed:, score:, reason: }



119
120
121
122
123
124
125
126
127
 
# File 'lib/ask/eval/assertions/deterministic.rb', line 119

def url(output, **_kwargs)
  uri = URI.parse(output.to_s.strip)
  passed = uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
  result(passed, score: passed ? 1.0 : 0.0,
         reason: passed ? "Output is a valid URL" :
                          "Output is not a valid URL")
rescue URI::InvalidURIError
  result(false, score: 0.0, reason: "Output is not a valid URL (parse error)")
end