Module: Silas::Eval::Grader

Defined in:
lib/silas/eval/grader.rb

Overview

Opt-in LLM judge for assert_rubric. Offline-skippable so the deterministic gate never depends on a live model unless you ask for it.

Class Method Summary collapse

Class Method Details

.available? ⇒ Boolean

Returns:

  • (Boolean) 


8
 
# File 'lib/silas/eval/grader.rb', line 8

def available? = ENV["SILAS_EVAL_OFFLINE"] != "1" && !grader.nil?

.builtin ⇒ Object 



15
16
17
18
19
20
21
22
 
# File 'lib/silas/eval/grader.rb', line 15

def builtin
  return nil if ENV["ANTHROPIC_API_KEY"].blank?

  lambda do |prompt|
    ::RubyLLM.chat(model: ENV["SILAS_EVAL_GRADER_MODEL"] || "claude-haiku-4-5-20251001")
             .ask(prompt).content.to_s
  end
end

.grade(prompt) ⇒ Object 



9
 
# File 'lib/silas/eval/grader.rb', line 9

def grade(prompt) = grader.call(prompt).to_s.strip

.grader ⇒ Object 



11
12
13
 
# File 'lib/silas/eval/grader.rb', line 11

def grader
  Silas.config.eval_grader || builtin
end

.prompt(transcript, criteria) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/silas/eval/grader.rb', line 24

def prompt(transcript, criteria)
  calls = transcript.invocations.map { |i| "#{i.tool_name}(#{i.arguments}) => #{i.result}" }.join("; ")
  <<~PROMPT
    You are grading an AI agent transcript. Reply with exactly one line: "PASS" or "FAIL: <reason>".
    Criteria: #{criteria}

    User: #{transcript.input}
    Tool calls: #{calls}
    Final answer: #{transcript.final_text}
  PROMPT
end