Class: Silas::Eval::Assertions::Context

Inherits:
Object
  • Object
show all
Defined in:
lib/silas/eval/assertions.rb

Overview

Assertions read ONLY the transcript rows; the block runs in this context. They collect failures rather than raising, so one eval reports all misses.

Constant Summary collapse

EXECUTED = 
%w[completed started in_doubt].freeze
MONEY = 
/(?<cur>[£$€])\s?(?<amt>\d{1,3}(?:,\d{3})*(?:\.\d+)?|\d+(?:\.\d+)?)/

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(transcript) ⇒ Context

Returns a new instance of Context.



12
13
14
15
16
 
# File 'lib/silas/eval/assertions.rb', line 12

def initialize(transcript)
  @t = transcript
  @failures = []
  @skips = []
end

Instance Attribute Details

#failures ⇒ Object (readonly)

Returns the value of attribute failures.



10
11
12
 
# File 'lib/silas/eval/assertions.rb', line 10

def failures
  @failures
end

#skips ⇒ Object (readonly)

Returns the value of attribute skips.



10
11
12
 
# File 'lib/silas/eval/assertions.rb', line 10

def skips
  @skips
end

Instance Method Details

#assert_answer_data(expected = :__unset, key: nil, value: :__unset, &pred) ⇒ Object

Structured (final_answer) assertions. With a Hash, expects the whole payload; with key/value, one field; with a block, a predicate over the payload. String keys — the payload is stored jsonb.



45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/silas/eval/assertions.rb', line 45

def assert_answer_data(expected = :__unset, key: nil, value: :__unset, &pred)
  data = @t.answer_data
  return check(false, "no structured answer (final_answer schema not set, or turn unfinished)") if data.nil?

  if pred
    check(pred.call(data), "answer_data predicate failed for #{data.inspect}")
  elsif key
    check(data[key.to_s] == value, "answer_data.#{key} expected #{value.inspect}, got #{data[key.to_s].inspect}")
  elsif expected != :__unset
    check(data == expected, "answer_data expected #{expected.inspect}, got #{data.inspect}")
  else
    check(true, nil) # presence alone
  end
end

#assert_approved(tool: nil) ⇒ Object 



70
71
72
73
 
# File 'lib/silas/eval/assertions.rb', line 70

def assert_approved(tool: nil)
  inv = tool ? @t.invocations_for(tool).last : @t.invocations.last
  check(inv&.approval_state == "approved", "expected #{tool || 'an invocation'} approved, state=#{inv&.approval_state.inspect}")
end

#assert_final_matches(matcher) ⇒ Object 



38
39
40
 
# File 'lib/silas/eval/assertions.rb', line 38

def assert_final_matches(matcher)
  check(matcher === @t.final_text, "final answer #{@t.final_text.inspect} does not match #{matcher.inspect}")
end

#assert_no_hallucinated_price(allowed: [], scale: [ 1, 100, 0.01 ]) ⇒ Object

No-hallucinated-price guard: every money amount in the final answer must trace to a number the agent actually saw (tool results or the user input), allowing pence<->pounds scaling.



63
64
65
66
67
68
 
# File 'lib/silas/eval/assertions.rb', line 63

def assert_no_hallucinated_price(allowed: [], scale: [ 1, 100, 0.01 ])
  stated = @t.final_text.scan(MONEY).map { |_c, a| a.delete(",").to_f }
  grounded = grounded_numbers + Array(allowed).map(&:to_f)
  bad = stated.reject { |n| grounded.any? { |g| scale.any? { |s| (g * s - n).abs < 0.005 } } }
  check(bad.empty?, "final answer states ungrounded amount(s) #{bad.inspect}; grounded=#{grounded.sort.inspect}")
end

#assert_no_tool_called(name = nil) ⇒ Object 



24
25
26
27
 
# File 'lib/silas/eval/assertions.rb', line 24

def assert_no_tool_called(name = nil)
  bad = name ? execed(name) : @t.invocations.select { |i| EXECUTED.include?(i.status) }
  check(bad.empty?, "expected no#{" #{name}" if name} tool execution, saw #{bad.map(&:tool_name)}")
end

#assert_parked(tool: nil) ⇒ Object 



75
76
77
78
79
 
# File 'lib/silas/eval/assertions.rb', line 75

def assert_parked(tool: nil)
  parked = @t.parked?
  parked &&= @t.invocations_for(tool).any? { |i| i.approval_state == "required" || i.status == "in_doubt" } if tool
  check(parked, "expected turn parked#{" on #{tool}" if tool}, status=#{@t.status}")
end

#assert_rubric(criteria) ⇒ Object

LLM-graded — opt-in; SKIPS offline (never fails the gate unless SILAS_EVAL_STRICT=1).



91
92
93
94
95
96
97
98
99
100
 
# File 'lib/silas/eval/assertions.rb', line 91

def assert_rubric(criteria)
  unless Grader.available?
    @skips << "assert_rubric skipped (no grader / offline)"
    return check(false, "assert_rubric strict skip (SILAS_EVAL_STRICT)") if ENV["SILAS_EVAL_STRICT"] == "1"

    return
  end
  verdict = Grader.grade(Grader.prompt(@t, criteria))
  check(verdict.start_with?("PASS"), "rubric FAIL: #{verdict}")
end

#assert_tool_arg(name, key, value = :__unset, &pred) ⇒ Object 



29
30
31
32
33
34
35
36
 
# File 'lib/silas/eval/assertions.rb', line 29

def assert_tool_arg(name, key, value = :__unset, &pred)
  inv = execed(name).last || @t.invocations_for(name).last
  return check(false, "#{name} was never called") unless inv

  got = inv.arguments[key.to_s]
  ok = pred ? pred.call(got) : got == value
  check(ok, "#{name}.#{key} expected #{pred ? 'predicate' : value.inspect}, got #{got.inspect}")
end

#assert_tool_called(name, times: nil) ⇒ Object 



18
19
20
21
22
 
# File 'lib/silas/eval/assertions.rb', line 18

def assert_tool_called(name, times: nil)
  n = execed(name).size
  check(times ? n == times : n.positive?,
        "expected #{name} called#{" #{times}x" if times}, saw #{n} (#{summary})")
end

#assert_turn_completed ⇒ Object 



81
82
83
 
# File 'lib/silas/eval/assertions.rb', line 81

def assert_turn_completed
  check(@t.completed?, "turn not completed (#{@t.status}/#{@t.turn.failure_reason})")
end

#assert_turn_failed(reason: nil) ⇒ Object 



85
86
87
88
 
# File 'lib/silas/eval/assertions.rb', line 85

def assert_turn_failed(reason: nil)
  ok = @t.status == "failed" && (reason.nil? || @t.turn.failure_reason == reason)
  check(ok, "expected failed#{"/#{reason}" if reason}, got #{@t.status}/#{@t.turn.failure_reason}")
end