Class: Ask::Eval::Recorder

Inherits:
Object
  • Object
show all
Defined in:
lib/ask/eval/recorder.rb

Overview

Records and replays LLM provider calls for regression testing.

Wraps an LLM provider's chat method to capture every request and response. In record mode, interactions are saved to YAML. In replay mode, recorded responses are returned instead of making real API calls.

Examples:

Recording

recorder = Ask::Eval::Recorder.new(test_name: "health_suite")
recorder.wrap(session)
session.run("Check health")
recorder.save  # test/recordings/health_suite/recording.yml

Replaying

recorder = Ask::Eval::Recorder.new(test_name: "health_suite", mode: :replay)
recorder.wrap(session)
session.run("Check health")  # uses recorded response, no API call

Constant Summary collapse

RECORDINGS_DIR = 
"test/recordings"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(test_name:, mode: nil, recordings_dir: nil) ⇒ Recorder

Returns a new instance of Recorder.



30
31
32
33
34
35
36
 
# File 'lib/ask/eval/recorder.rb', line 30

def initialize(test_name:, mode: nil, recordings_dir: nil)
  @test_name = test_name
  @mode = (mode || detect_mode).to_sym
  @recordings_dir = recordings_dir || RECORDINGS_DIR
  @interactions = []
  @replay_queue = []
end

Instance Attribute Details

#mode ⇒ Object (readonly)

Returns the value of attribute mode.



28
29
30
 
# File 'lib/ask/eval/recorder.rb', line 28

def mode
  @mode
end

#test_name ⇒ Object (readonly)

Returns the value of attribute test_name.



28
29
30
 
# File 'lib/ask/eval/recorder.rb', line 28

def test_name
  @test_name
end

Instance Method Details

#record_call(args:, kwargs:, result_data:) ⇒ Object

Record a provider call with serialized result data.



68
69
70
71
72
73
74
 
# File 'lib/ask/eval/recorder.rb', line 68

def record_call(args:, kwargs:, result_data:)
  @interactions << {
    messages: scrub_messages(args.first),
    model: kwargs[:model],
    result: result_data
  }
end

#recording? ⇒ Boolean

Returns:

  • (Boolean) 


38
 
# File 'lib/ask/eval/recorder.rb', line 38

def recording? = @mode == :record

#recording_path ⇒ Object 



137
138
139
 
# File 'lib/ask/eval/recorder.rb', line 137

def recording_path
  File.join(@recordings_dir, sanitize(@test_name), "recording.yml")
end

#replay_as_message ⇒ Object

Replay the next recorded call as an Ask::Message.



77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/ask/eval/recorder.rb', line 77

def replay_as_message
  load_recording_if_needed

  entry = @replay_queue.shift
  unless entry
    raise "No recorded interaction available. Delete #{recording_path} and re-record."
  end

  result = entry["result"]

  if result["type"] == "stream"
    stream = Ask::Stream.new
    (result["chunks"] || []).each do |c|
      stream.add(Ask::Chunk.new(
        content: c["content"],
        tool_calls: c["tool_calls"],
        finish_reason: c["finish_reason"],
        thinking: c["thinking"]
      ))
    end
    stream.finish!
    stream
  else
    Ask::Message.new(
      role: :assistant,
      content: result["content"],
      tool_calls: result["tool_calls"],
      metadata: result["metadata"] || {}
    )
  end
end

#replay_call ⇒ Object

Replay the next recorded call (returns raw data).



110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/ask/eval/recorder.rb', line 110

def replay_call
  load_recording_if_needed

  entry = @replay_queue.shift
  unless entry
    raise "No recorded interaction available. " \
          "The test may have changed since recording. " \
          "Delete #{recording_path} and re-record."
  end
  entry["result"]
end

#replaying? ⇒ Boolean

Returns:

  • (Boolean) 


39
 
# File 'lib/ask/eval/recorder.rb', line 39

def replaying? = @mode == :replay

#save ⇒ Object

Save recorded interactions to disk.



123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/ask/eval/recorder.rb', line 123

def save
  return unless recording?
  return if @interactions.empty?

  dir = File.join(@recordings_dir, sanitize(@test_name))
  FileUtils.mkdir_p(dir)

  File.write(recording_path, JSON.pretty_generate({
    "test" => @test_name,
    "recorded_at" => Time.now.utc.iso8601,
    "interactions" => @interactions
  }))
end

#wrap(session) ⇒ Object

Wrap a Session's provider to record or replay LLM calls.



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/ask/eval/recorder.rb', line 42

def wrap(session)
  recorder = self
  load_recording if replaying?

  session.chat.define_singleton_method(:build_provider) do
    original = super()

    proxy = Object.new
    proxy.define_singleton_method(:chat) do |*args, **kwargs, &block|
      if recorder.replaying?
        recorder.replay_as_message
      else
        result = original.chat(*args, **kwargs, &block)
        recorder.record_call(args: args, kwargs: kwargs, result_data: serialize(result))
        result
      end
    end

    proxy.define_singleton_method(:respond_to?) { |name, *a| original.respond_to?(name, *a) || super(name, *a) }
    proxy
  end

  session.chat.instance_variable_set(:@provider, nil)
end