Class: Silas::Eval::DSL

Inherits:
Object
  • Object
show all
Defined in:
lib/silas/eval/dsl.rb

Overview

Define-time recorder for the scenario DSL.

Instance Method Summary collapse

Constructor Details

#initialize(name, tags) ⇒ DSL

Returns a new instance of DSL.



5
6
7
8
9
10
11
12
13
14
 
# File 'lib/silas/eval/dsl.rb', line 5

def initialize(name, tags)
  @name = name
  @tags = tags
  @mode = :fake
  @steps = {}
  @stubs = {}
  @approvals = []
  @metadata = {}
  @max_steps = nil
end

Instance Method Details

#approve(tool:) ⇒ Object 



20
 
# File 'lib/silas/eval/dsl.rb', line 20

def approve(tool:) = (@approvals << tool.to_s)

#expect(&block) ⇒ Object 



48
 
# File 'lib/silas/eval/dsl.rb', line 48

def expect(&block) = (@assertions = block)

#input(text) ⇒ Object 



16
 
# File 'lib/silas/eval/dsl.rb', line 16

def input(text)  = (@input = text)

#max_steps(n) ⇒ Object 



19
 
# File 'lib/silas/eval/dsl.rb', line 19

def max_steps(n) = (@max_steps = n)

#metadata(h) ⇒ Object 



17
 
# File 'lib/silas/eval/dsl.rb', line 17

def metadata(h)  = (@metadata = h)

#mode(m) ⇒ Object 



18
 
# File 'lib/silas/eval/dsl.rb', line 18

def mode(m)      = (@mode = m)

#on_step(index, text: nil, call: nil, calls: [], data: nil) ⇒ Object

on_step(0, text:, call: arguments:, calls: [ …, … ], data: …) data: scripts a STRUCTURED answer (the final_answer schema case) — it becomes the same "type"=>"structured" block the real adapter persists, so assert_answer_data reads it back through Turn#answer_data. Keys are stringified exactly as a JSON-parsed model response would be.



27
28
29
30
31
32
33
34
35
36
 
# File 'lib/silas/eval/dsl.rb', line 27

def on_step(index, text: nil, call: nil, calls: [], data: nil)
  tcs = (calls + [ call ].compact).each_with_index.map do |c, n|
    Silas::Adapters::ToolCall.new(id: "eval_s#{index}_#{n}", name: c[:name].to_s,
                                 arguments: (c[:arguments] || {}).stringify_keys)
  end
  blocks = []
  blocks << { "type" => "structured", "data" => data.deep_stringify_keys } if data
  blocks << { "type" => "text", "text" => text } if text
  @steps[index] = { blocks: blocks, tool_calls: tcs }
end

#stub_tool(name, effect_mode: :at_most_once, approval: :never, &body) ⇒ Object

Override a real tool with a stub for a side-effect-free eval.



39
40
41
42
43
44
45
46
 
# File 'lib/silas/eval/dsl.rb', line 39

def stub_tool(name, effect_mode: :at_most_once, approval: :never, &body)
  obj = Object.new
  obj.define_singleton_method(:effect_mode) { effect_mode }
  obj.define_singleton_method(:approval_policy) { approval }
  obj.singleton_class.attr_accessor :session
  obj.define_singleton_method(:call, &body)
  @stubs[name.to_s] = obj
end

#to_scenario ⇒ Object

Raises:



50
51
52
53
54
55
56
57
 
# File 'lib/silas/eval/dsl.rb', line 50

def to_scenario
  raise Silas::Error, "eval #{@name.inspect} needs an input" unless @input
  raise Silas::Error, "eval #{@name.inspect} needs an expect block" unless @assertions

  Scenario.new(name: @name, input: @input, mode: @mode, steps: @steps,
               stubs: @stubs, approvals: @approvals, metadata: @metadata,
               max_steps: @max_steps, tags: @tags, assertions: @assertions)
end