Class: PromptCanary::Recorder

Inherits:
Object
  • Object
show all
Defined in:
lib/prompt_canary/recorder.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(storage:) ⇒ Recorder

Returns a new instance of Recorder.



7
8
9
 
# File 'lib/prompt_canary/recorder.rb', line 7

def initialize(storage:)
  @storage = storage
end

Instance Attribute Details

#storage ⇒ Object (readonly)

Returns the value of attribute storage.



5
6
7
 
# File 'lib/prompt_canary/recorder.rb', line 5

def storage
  @storage
end

Instance Method Details

#error_rate(prompt:, version:, over:) ⇒ Object 



20
21
22
23
24
25
26
 
# File 'lib/prompt_canary/recorder.rb', line 20

def error_rate(prompt:, version:, over:)
  records = @storage.read_recent(prompt: prompt, version: version, limit: over)
  return 0.0 if records.empty?

  errored = records.count { |r| !r[:error].nil? }
  (errored.to_f / records.length).round(2)
end

#latency_p95(prompt:, version:, over:) ⇒ Object 



11
12
13
14
15
16
17
18
 
# File 'lib/prompt_canary/recorder.rb', line 11

def latency_p95(prompt:, version:, over:)
  records = @storage.read_recent(prompt: prompt, version: version, limit: over)
  return 0 if records.empty?

  latencies = records.map { |r| r[:latency_ms] }.sort
  index = (latencies.length * 0.95).ceil - 1
  latencies[index]
end

#record(prompt:, version:, telemetry:) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
 
# File 'lib/prompt_canary/recorder.rb', line 44

def record(prompt:, version:, telemetry:)
  @storage.write(
    prompt: prompt,
    version: version.name,
    latency_ms: telemetry[:latency_ms],
    tokens: telemetry[:tokens],
    error: telemetry[:error],
    recorded_at: Time.now
  )
end

#stats(prompt:, version:, over:) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/prompt_canary/recorder.rb', line 28

def stats(prompt:, version:, over:)
  records = @storage.read_recent(prompt: prompt, version: version, limit: over)
  return { call_count: 0, error_rate: 0.0, latency_p95: nil, last_called_at: nil } if records.empty?

  latencies   = records.map { |r| r[:latency_ms] }.compact.sort
  error_count = records.count { |r| !r[:error].nil? }
  p95_index   = (latencies.size * 0.95).ceil - 1

  {
    call_count: records.size,
    error_rate: (error_count.to_f / records.size).round(2),
    latency_p95: latencies[p95_index],
    last_called_at: records.last[:recorded_at]
  }
end