Module: Ask::Eval::DSL

Defined in:
lib/ask/eval/dsl.rb

Overview

Minitest DSL mixin. Include this in your test class to get all ask-eval assertion methods, including session evaluation.

Examples:

class MyEvalTest < Minitest::Test
  include Ask::Eval::DSL

  test "response quality" do
    assert_faithful my_response, context: docs
    assert_contains my_response, "policy"
  end

  test "agent behavior" do
    eval_session(model: "gpt-4o", tools: [Bash]) do |r|
      r.run("Check health")
      assert_tool_called "bash"
      assert_cost_under 0.01
    end
  end
end

Instance Method Summary collapse

Instance Method Details

#assert_contains(output, value, msg = nil) ⇒ Object

--- Deterministic Assertions ---



108
109
110
111
 
# File 'lib/ask/eval/dsl.rb', line 108

def assert_contains(output, value, msg = nil)
  result = Assertions::Deterministic.contains(output, value: value)
  assert result[:passed], msg || result[:reason]
end

#assert_correctness(output, expected:, model: nil, msg: nil) ⇒ Object 



204
205
206
207
208
209
210
 
# File 'lib/ask/eval/dsl.rb', line 204

def assert_correctness(output, expected:, model: nil, msg: nil)
  tc = TestCase.new(actual_output: output, expected_output: expected)
  judge = Judges::Correctness.new(model: model)
  result = judge.call(tc)
  assert result.passed, msg || "Correctness check failed: #{result.reason}"
  log_cost(result) if Ask::Eval.configuration.track_cost
end

#assert_cost_under(max_cost, msg = nil) ⇒ Object

Assert that no more than max_cost was spent on LLM calls.

Parameters:

  • max_cost (Float) —

    maximum allowed cost in USD

  • msg (String, nil) (defaults to: nil) —

    custom failure message



87
88
89
90
91
92
93
94
 
# File 'lib/ask/eval/dsl.rb', line 87

def assert_cost_under(max_cost, msg = nil)
  session_eval = @_ask_eval_session
  return flunk(msg || "No session eval context") unless session_eval

  cost = session_eval.total_cost
  assert cost <= max_cost,
         msg || "Expected cost <= #{max_cost}, got $#{'%.6f' % cost}"
end

#assert_email(output, msg = nil) ⇒ Object 



163
164
165
166
 
# File 'lib/ask/eval/dsl.rb', line 163

def assert_email(output, msg = nil)
  result = Assertions::Deterministic.email(output)
  assert result[:passed], msg || result[:reason]
end

#assert_ends_with(output, suffix, msg = nil) ⇒ Object 



138
139
140
141
 
# File 'lib/ask/eval/dsl.rb', line 138

def assert_ends_with(output, suffix, msg = nil)
  result = Assertions::Deterministic.ends_with(output, suffix: suffix)
  assert result[:passed], msg || result[:reason]
end

#assert_equals(output, value, msg = nil) ⇒ Object 



143
144
145
146
 
# File 'lib/ask/eval/dsl.rb', line 143

def assert_equals(output, value, msg = nil)
  result = Assertions::Deterministic.equals(output, value: value)
  assert result[:passed], msg || result[:reason]
end

#assert_faithful(output, context:, model: nil, threshold: 0.7, msg: nil) ⇒ Object

--- LLM Judge Assertions ---



170
171
172
173
174
175
176
177
 
# File 'lib/ask/eval/dsl.rb', line 170

def assert_faithful(output, context:, model: nil, threshold: 0.7, msg: nil)
  tc = TestCase.new(actual_output: output, context: context)
  judge = Judges::Faithful.new(model: model)
  result = judge.call(tc)
  passed = result.score >= threshold
  assert passed, msg || "Faithfulness check failed: #{result.reason} (score: #{result.score})"
  log_cost(result) if Ask::Eval.configuration.track_cost
end

#assert_json(output, msg = nil) ⇒ Object 



123
124
125
126
 
# File 'lib/ask/eval/dsl.rb', line 123

def assert_json(output, msg = nil)
  result = Assertions::Deterministic.is_json(output)
  assert result[:passed], msg || result[:reason]
end

#assert_max_length(output, max, msg = nil) ⇒ Object 



153
154
155
156
 
# File 'lib/ask/eval/dsl.rb', line 153

def assert_max_length(output, max, msg = nil)
  result = Assertions::Deterministic.max_length(output, max: max)
  assert result[:passed], msg || result[:reason]
end

#assert_max_tokens(output, max, msg = nil) ⇒ Object 



128
129
130
131
 
# File 'lib/ask/eval/dsl.rb', line 128

def assert_max_tokens(output, max, msg = nil)
  result = Assertions::Deterministic.max_tokens(output, max: max)
  assert result[:passed], msg || result[:reason]
end

#assert_min_length(output, min, msg = nil) ⇒ Object 



148
149
150
151
 
# File 'lib/ask/eval/dsl.rb', line 148

def assert_min_length(output, min, msg = nil)
  result = Assertions::Deterministic.min_length(output, min: min)
  assert result[:passed], msg || result[:reason]
end

#assert_not_contains(output, value, msg = nil) ⇒ Object 



113
114
115
116
 
# File 'lib/ask/eval/dsl.rb', line 113

def assert_not_contains(output, value, msg = nil)
  result = Assertions::Deterministic.not_contains(output, value: value)
  assert result[:passed], msg || result[:reason]
end

#assert_not_hallucinating(output, context:, model: nil, threshold: 0.7, msg: nil) ⇒ Object 



179
180
181
182
183
184
185
186
 
# File 'lib/ask/eval/dsl.rb', line 179

def assert_not_hallucinating(output, context:, model: nil, threshold: 0.7, msg: nil)
  tc = TestCase.new(actual_output: output, context: context)
  judge = Judges::Hallucination.new(model: model)
  result = judge.call(tc)
  passed = result.score >= threshold
  assert passed, msg || "Hallucination check failed: #{result.reason} (score: #{result.score})"
  log_cost(result) if Ask::Eval.configuration.track_cost
end

#assert_regex(output, pattern, msg = nil) ⇒ Object 



118
119
120
121
 
# File 'lib/ask/eval/dsl.rb', line 118

def assert_regex(output, pattern, msg = nil)
  result = Assertions::Deterministic.regex(output, pattern: pattern)
  assert result[:passed], msg || result[:reason]
end

#assert_starts_with(output, prefix, msg = nil) ⇒ Object 



133
134
135
136
 
# File 'lib/ask/eval/dsl.rb', line 133

def assert_starts_with(output, prefix, msg = nil)
  result = Assertions::Deterministic.starts_with(output, prefix: prefix)
  assert result[:passed], msg || result[:reason]
end

#assert_tool_called(tool_name, msg = nil) ⇒ Object

Assert that a specific tool was called during the session.

Parameters:

  • tool_name (String) —

    tool name (e.g. "bash", "read")

  • msg (String, nil) (defaults to: nil) —

    custom failure message



74
75
76
77
78
79
80
81
 
# File 'lib/ask/eval/dsl.rb', line 74

def assert_tool_called(tool_name, msg = nil)
  session_eval = @_ask_eval_session
  return flunk(msg || "No session eval context") unless session_eval

  assert session_eval.tool_called?(tool_name),
         msg || "Expected tool #{tool_name.inspect} to have been called. " \
                "Tools called: #{session_eval.tool_names.inspect}"
end

#assert_url(output, msg = nil) ⇒ Object 



158
159
160
161
 
# File 'lib/ask/eval/dsl.rb', line 158

def assert_url(output, msg = nil)
  result = Assertions::Deterministic.url(output)
  assert result[:passed], msg || result[:reason]
end

#eval_output(output, recorder: nil, scenario: nil) ⇒ Object

Eval an agent session with recording support. This is the low-level method that eval_session delegates to.



100
101
102
103
104
 
# File 'lib/ask/eval/dsl.rb', line 100

def eval_output(output, recorder: nil, scenario: nil, **)
  if recorder&.recording?
    recorder.record_assertion(name: caller_locations(1, 1)&.label, passed: true, score: 1.0, reason: "recorded")
  end
end

#eval_session(model:, tools: [], system_prompt: nil, recordings_dir: nil, **session_opts) {|SessionEval| ... } ⇒ Object

Evaluate an agent session within a test.

The session runs with regression recording enabled. On first run, interactions are recorded to test/recordings/<test_name>/. On subsequent runs with ASK_EVAL_MODE=replay, recorded responses are used instead of real LLM calls.

Parameters:

  • model (String) —

    model identifier

  • tools (Array) (defaults to: []) —

    tool classes or instances

  • system_prompt (String, nil) (defaults to: nil) —

    optional system prompt

  • recordings_dir (String, nil) (defaults to: nil) —

    custom recordings directory

Yields:

  • (SessionEval) —

    yields a session eval for running and asserting



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/ask/eval/dsl.rb', line 40

def eval_session(model:, tools: [], system_prompt: nil, recordings_dir: nil, **session_opts)
  test_name = self.name

  recorder = Recorder.new(
    test_name: test_name,
    recordings_dir: recordings_dir
  )

  session = Ask::Agent::Session.new(
    model: model,
    tools: tools,
    system_prompt: system_prompt,
    **session_opts
  )

  unless recorder.replaying?
    recorder.wrap(session)
  end

  eval_runner = SessionEval.new(session, recorder: recorder)

  # Make session eval available to assert_tool_called / assert_cost_under
  @_ask_eval_session = eval_runner

  yield eval_runner
ensure
  @_ask_eval_session = nil
  recorder&.save
end

#refute_bias(output, model: nil, msg: nil) ⇒ Object 



188
189
190
191
192
193
194
 
# File 'lib/ask/eval/dsl.rb', line 188

def refute_bias(output, model: nil, msg: nil)
  tc = TestCase.new(actual_output: output)
  judge = Judges::Bias.new(model: model)
  result = judge.call(tc)
  assert result.passed, msg || "Bias detected: #{result.reason}"
  log_cost(result) if Ask::Eval.configuration.track_cost
end

#refute_toxicity(output, model: nil, msg: nil) ⇒ Object 



196
197
198
199
200
201
202
 
# File 'lib/ask/eval/dsl.rb', line 196

def refute_toxicity(output, model: nil, msg: nil)
  tc = TestCase.new(actual_output: output)
  judge = Judges::Toxicity.new(model: model)
  result = judge.call(tc)
  assert result.passed, msg || "Toxicity detected: #{result.reason}"
  log_cost(result) if Ask::Eval.configuration.track_cost
end