7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
|
# File 'lib/aispec/core/assertions/judge.rb', line 7
def self.register!
Assertion.register(:judge) do |response, context, options|
judge_prompt = options[:prompt] || options[:criteria] || "Evaluate if the response fulfills the contract requirements."
threshold = (options[:threshold] || options[:min_score] || 0.80).to_f
target_output = response[:output].to_s
input_context = context[:input] ? "Input: #{context[:input]}\n" : ""
evaluation_instructions = <<~SYSTEM
You are an impartial AI behavioral evaluator judge.
Evaluate the following AI response against the criteria below.
Criteria:
#{judge_prompt}
#{input_context}
AI Response to evaluate:
\"\"\"
#{target_output}
\"\"\"
Respond ONLY in valid JSON format with:
{
"score": <float between 0.0 and 1.0>,
"reason": "<brief justification>"
}
SYSTEM
judge_provider_name = options[:provider] || AISpec.configuration.judge_provider
judge_model = options[:model] || AISpec.configuration.judge_model
provider = Providers::Base.get(judge_provider_name).new(model: judge_model)
eval_result = provider.generate(evaluation_instructions)
begin
cleaned = eval_result[:output].to_s.sub(/^```json\s*/i, "").sub(/^```\s*/, "").sub(/\s*```$/, "").strip
parsed = JSON.parse(cleaned)
score = (parsed["score"] || parsed[:score] || 0.0).to_f
reason = parsed["reason"] || parsed[:reason] || "No explanation provided"
passed = score >= threshold
[
passed,
"Judge score #{sprintf('%.2f', score)} (threshold #{threshold}): #{reason}",
{ score: score, threshold: threshold, reason: reason }
]
rescue JSON::ParserError => _e
match = eval_result[:output].to_s.match(/(?:score|rating)?\s*[:=]?\s*(0\.\d+|1\.0|\d+)/i)
score = match ? match[1].to_f : 0.5
score = score / 10.0 if score > 1.0
passed = score >= threshold
[
passed,
"Judge score #{sprintf('%.2f', score)} (threshold #{threshold})",
{ score: score, threshold: threshold }
]
end
end
Assertion.register(:confidence) do |response, context, options|
threshold = (options[:min] || options[:threshold] || 0.8).to_f
options_with_judge = options.merge(
prompt: "Evaluate confidence and factual accuracy of the response.",
threshold: threshold
)
Assertion.evaluate(:judge, response, context, options_with_judge)
end
end
|