Module: Silas::Eval::Runner

Defined in:
lib/silas/eval/runner.rb

Overview

Discovers *_eval.rb files, runs each scenario in isolation against the real loop, reports, and returns a pass/fail boolean (the rake task exits 1 on false).

Class Method Summary collapse

Class Method Details

.report(results) ⇒ Object 



31
32
33
34
35
 
# File 'lib/silas/eval/runner.rb', line 31

def report(results)
  results.each { |r| puts r.line }
  failing = results.count { |r| !(r.pass? || r.skipped?) }
  puts "\nsilas eval: #{results.size} scenarios, #{failing} failing, #{results.count(&:skipped?)} skipped"
end

.run(dir:, pattern: nil, mode: nil, root: Rails.root) ⇒ Object 



8
9
10
11
12
13
14
15
16
 
# File 'lib/silas/eval/runner.rb', line 8

def run(dir:, pattern: nil, mode: nil, root: Rails.root)
  Eval.reset!
  Dir[File.join(dir, "**/*_eval.rb")].sort.each { |f| load f }
  picked = Eval.scenarios.select { |s| pattern.nil? || s.name.match?(pattern) }

  results = picked.map { |s| run_one(s.with_mode(mode), root) }
  report(results)
  results.all? { |r| r.pass? || r.skipped? }
end

.run_one(scenario, root = Rails.root) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/silas/eval/runner.rb', line 18

def run_one(scenario, root = Rails.root)
  if scenario.real? && ENV["ANTHROPIC_API_KEY"].to_s.empty?
    return Result.skipped(scenario, "real mode, no ANTHROPIC_API_KEY")
  end

  transcript = Driver.drive(scenario, root: root)
  ctx = Assertions::Context.new(transcript)
  ctx.instance_exec(&scenario.assertions)
  Result.new(scenario, ctx.failures, ctx.skips)
rescue StandardError => e
  Result.errored(scenario, e)
end