Class: Ask::Eval::Recorder

Inherits:
Object
  • Object
show all
Defined in:
lib/ask/eval/recorder.rb

Overview

Records and replays LLM provider calls for regression testing.

Wraps an LLM provider's chat method to capture every request and response. In record mode, interactions are saved to YAML. In replay mode, recorded responses are returned instead of making real API calls.

Examples:

Recording

recorder = Ask::Eval::Recorder.new(test_name: "health_suite")
recorder.wrap(session)
session.run("Check health")
recorder.save  # test/recordings/health_suite/recording.yml

Replaying

recorder = Ask::Eval::Recorder.new(test_name: "health_suite", mode: :replay)
recorder.wrap(session)
session.run("Check health")  # uses recorded response, no API call

Constant Summary collapse

RECORDINGS_DIR = 
"test/recordings"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(test_name:, mode: nil, recordings_dir: nil) ⇒ Recorder

Returns a new instance of Recorder.



30
31
32
33
34
35
36
 
# File 'lib/ask/eval/recorder.rb', line 30

def initialize(test_name:, mode: nil, recordings_dir: nil)
  @test_name = test_name
  @mode = (mode || detect_mode).to_sym
  @recordings_dir = recordings_dir || RECORDINGS_DIR
  @interactions = []
  @replay_queue = []
end

Instance Attribute Details

#mode ⇒ Object (readonly)

Returns the value of attribute mode.



28
29
30
 
# File 'lib/ask/eval/recorder.rb', line 28

def mode
  @mode
end

#test_name ⇒ Object (readonly)

Returns the value of attribute test_name.



28
29
30
 
# File 'lib/ask/eval/recorder.rb', line 28

def test_name
  @test_name
end

Instance Method Details

#record_call(args:, kwargs:, result_data:) ⇒ Object

Record a provider call with serialized result data.



68
69
70
71
72
73
74
75
 
# File 'lib/ask/eval/recorder.rb', line 68

def record_call(args:, kwargs:, result_data:)
  @interactions << {
    type: "provider",
    messages: scrub_messages(args.first),
    model: kwargs[:model],
    result: result_data
  }
end

#record_tool_call(name:, args:, result_data:) ⇒ Object

Record a tool execution with serialized result data. Tool calls are interleaved with provider calls in the same order they happened, so a multi-turn agent run replays faithfully — tool results are replayed, not re-executed.



81
82
83
84
85
86
87
88
 
# File 'lib/ask/eval/recorder.rb', line 81

def record_tool_call(name:, args:, result_data:)
  @interactions << {
    type: "tool",
    name: name,
    args: args,
    result: result_data
  }
end

#recording? ⇒ Boolean

Returns:

  • (Boolean) 


38
 
# File 'lib/ask/eval/recorder.rb', line 38

def recording? = @mode == :record

#recording_path ⇒ Object 



182
183
184
 
# File 'lib/ask/eval/recorder.rb', line 182

def recording_path
  File.join(@recordings_dir, sanitize(@test_name), "recording.yml")
end

#replay_as_message ⇒ Object

Replay the next recorded provider call as an Ask::Message.



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/ask/eval/recorder.rb', line 91

def replay_as_message
  load_recording_if_needed

  entry = @replay_queue.shift
  unless entry
    raise "No recorded interaction available. Delete #{recording_path} and re-record."
  end

  if entry["type"] == "tool"
    raise "Replay diverged: expected a provider call, but the next recorded interaction is a tool call. " \
          "The run took a different path than the recording. Delete #{recording_path} and re-record."
  end

  result = entry["result"]

  if result["type"] == "stream"
    stream = Ask::Stream.new
    (result["chunks"] || []).each do |c|
      stream.add(Ask::Chunk.new(
        content: c["content"],
        tool_calls: c["tool_calls"],
        finish_reason: c["finish_reason"],
        thinking: c["thinking"]
      ))
    end
    stream.finish!
    stream
  else
    Ask::Message.new(
      role: :assistant,
      content: result["content"],
      tool_calls: result["tool_calls"],
      metadata: result["metadata"] || {}
    )
  end
end

#replay_call ⇒ Object

Replay the next recorded call (returns raw data).



129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/ask/eval/recorder.rb', line 129

def replay_call
  load_recording_if_needed

  entry = @replay_queue.shift
  unless entry
    raise "No recorded interaction available. " \
          "The test may have changed since recording. " \
          "Delete #{recording_path} and re-record."
  end
  entry["result"]
end

#replay_tool_call ⇒ Object

Replay the next recorded tool execution, returning its original Ask::Result without executing the tool again.



143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/ask/eval/recorder.rb', line 143

def replay_tool_call
  load_recording_if_needed

  entry = @replay_queue.shift
  unless entry
    raise "No recorded tool interaction available. Delete #{recording_path} and re-record."
  end
  unless entry["type"] == "tool"
    raise "Replay diverged: expected a tool call, but the next recorded interaction is a provider call. " \
          "The run took a different path than the recording. Delete #{recording_path} and re-record."
  end

  result = entry["result"]
  if result["type"] == "result"
    if result["ok"]
      Ask::Result.ok(data: result["output"], metadata: result["metadata"] || {})
    else
      Ask::Result.error(message: result["error"] || "", metadata: result["metadata"] || {})
    end
  else
    result["data"]
  end
end

#replaying? ⇒ Boolean

Returns:

  • (Boolean) 


39
 
# File 'lib/ask/eval/recorder.rb', line 39

def replaying? = @mode == :replay

#save ⇒ Object

Save recorded interactions to disk.



168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/ask/eval/recorder.rb', line 168

def save
  return unless recording?
  return if @interactions.empty?

  dir = File.join(@recordings_dir, sanitize(@test_name))
  FileUtils.mkdir_p(dir)

  File.write(recording_path, JSON.pretty_generate({
    "test" => @test_name,
    "recorded_at" => Time.now.utc.iso8601,
    "interactions" => @interactions
  }))
end

#wrap(session) ⇒ Object

Wrap a Session's provider to record or replay LLM calls.



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/ask/eval/recorder.rb', line 42

def wrap(session)
  recorder = self
  load_recording if replaying?

  session.chat.define_singleton_method(:build_provider) do
    original = super()

    proxy = Object.new
    proxy.define_singleton_method(:chat) do |*args, **kwargs, &block|
      if recorder.replaying?
        recorder.replay_as_message
      else
        result = original.chat(*args, **kwargs, &block)
        recorder.record_call(args: args, kwargs: kwargs, result_data: serialize(result))
        result
      end
    end

    proxy.define_singleton_method(:respond_to?) { |name, *a| original.respond_to?(name, *a) || super(name, *a) }
    proxy
  end

  session.chat.instance_variable_set(:@provider, nil)
end