Class: CompletionKit::Run

Inherits:
ApplicationRecord show all
Includes:
Taggable, Turbo::Broadcastable
Defined in:
app/models/completion_kit/run.rb

Constant Summary collapse

STATUSES = 
%w[pending running completed failed].freeze
INSERT_BATCH_SIZE = 
1000

Constants inherited from ApplicationRecord

ApplicationRecord::TenantScopedUniquenessValidator

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Taggable

#tag_names, #tag_names=

Instance Attribute Details

#avg_score ⇒ Object 



193
194
195
196
197
198
199
200
 
# File 'app/models/completion_kit/run.rb', line 193

def avg_score
  return @avg_score if defined?(@avg_score)

  scores = reviews_for_summary.map(&:ai_score).compact.map(&:to_f)
  return nil if scores.empty?

  (scores.sum / scores.length).round(2)
end

#check_pass_rate ⇒ Object 



220
221
222
223
224
225
226
227
228
 
# File 'app/models/completion_kit/run.rb', line 220

def check_pass_rate
  return @check_pass_rate if defined?(@check_pass_rate)

  resolved = reviews_for_summary.reject { |r| r.passed.nil? }
  return nil if resolved.empty?

  passed = resolved.count { |r| r.passed == true }
  (passed.to_f / resolved.length).round(2)
end

#metric_averages ⇒ Object 



202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'app/models/completion_kit/run.rb', line 202

def metric_averages
  return @metric_averages if defined?(@metric_averages)

  reviews_for_summary.group_by(&:metric_name).filter_map do |name, reviews|
    scored = reviews.select { |r| r.ai_score.present? }
    if scored.any?
      scores = scored.map { |r| r.ai_score.to_f }
      { name: name, avg: (scores.sum / scores.length).round(1) }
    else
      resolved = reviews.reject { |r| r.passed.nil? }
      next if resolved.empty?

      passed = resolved.count { |r| r.passed == true }
      { name: name, kind: "check", pass_rate: (passed.to_f / resolved.length).round(2) }
    end
  end
end

#response_count ⇒ Object 



187
188
189
190
191
 
# File 'app/models/completion_kit/run.rb', line 187

def response_count
  return @response_count if defined?(@response_count)

  responses.size
end

Class Method Details

.display_scoped ⇒ Object 



30
31
32
33
 
# File 'app/models/completion_kit/run.rb', line 30

def self.display_scoped
  filter = CompletionKit.config.runs_display_scope
  filter ? all.instance_exec(&filter) : all
end

.preload_summaries(runs) ⇒ Object

Batch-compute the list-view summaries (response count, avg score, check pass rate, per-metric averages) for a set of runs in a constant number of grouped queries, injecting the results so the index never loads a single response or review object. Mirrors the per-run reader methods exactly.



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'app/models/completion_kit/run.rb', line 43

def self.preload_summaries(runs)
  runs = runs.to_a
  return runs if runs.empty?

  run_ids = runs.map(&:id)
  counts = Response.where(run_id: run_ids).group(:run_id).count

  run_col = Arel.sql("completion_kit_responses.run_id")
  base = Review.joins(:response).where(completion_kit_responses: {run_id: run_ids})

  run_rows = base.group(run_col).pluck(
    run_col,
    Arel.sql("AVG(ai_score)"),
    Arel.sql("COUNT(passed)"),
    Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
  )
  run_stats = run_rows.each_with_object({}) do |(rid, avg, resolved, passed), h|
    h[rid] = {avg: avg, resolved: resolved.to_i, passed: passed.to_i}
  end

  metric_rows = base.group(run_col, :metric_name).pluck(
    run_col,
    :metric_name,
    Arel.sql("AVG(ai_score)"),
    Arel.sql("COUNT(ai_score)"),
    Arel.sql("COUNT(passed)"),
    Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
  )
  metrics_by_run = metric_rows.group_by(&:first)

  runs.each do |run|
    run.response_count = counts.fetch(run.id, 0)

    stats = run_stats[run.id]
    run.avg_score = stats && stats[:avg] ? stats[:avg].to_f.round(2) : nil
    run.check_pass_rate = stats && stats[:resolved] > 0 ? (stats[:passed].to_f / stats[:resolved]).round(2) : nil

    run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, resolved, passed)|
      if scored.to_i > 0
        {name: name, avg: avg.to_f.round(1)}
      elsif resolved.to_i > 0
        {name: name, kind: "check", pass_rate: (passed.to_i.to_f / resolved.to_i).round(2)}
      end
    end
  end
  runs
end

.visible_run_ids ⇒ Object 



35
36
37
 
# File 'app/models/completion_kit/run.rb', line 35

def self.visible_run_ids
  display_scoped.select(:id)
end

Instance Method Details

#as_json(options = {}) ⇒ Object 



459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
 
# File 'app/models/completion_kit/run.rb', line 459

def as_json(options = {})
  snap = progress_snapshot
  {
    id: id, name: name, status: status, prompt_id: prompt_id,
    dataset_id: dataset_id, judge_model: judge_model, temperature: temperature,
    output_column: output_column,
    expected_column: expected_column,
    created_at: created_at, updated_at: updated_at,
    responses_count: responses.count, avg_score: avg_score,
    check_pass_rate: check_pass_rate,
    progress_current: snap[:generated_done],
    progress_total: snap[:generated_total],
    progress: {
      generated: { done: snap[:generated_done], total: snap[:generated_total], failed: snap[:generated_failed] },
      judged:    { done: snap[:judged_done],    total: snap[:judged_total],    failed: snap[:judged_failed] }
    },
    failed_response_ids: responses.where(status: "failed").pluck(:id),
    failure_summary: failure_summary,
    error_message: error_message,
    metric_ids: metric_ids,
    tags: tags.as_json
  }
end

#broadcast_actions ⇒ Object 



507
508
509
510
511
512
513
 
# File 'app/models/completion_kit/run.rb', line 507

def broadcast_actions
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_actions",
    html: render_engine_partial("completion_kit/runs/actions", run: self)
  )
end

#broadcast_clear_responses ⇒ Object 



523
524
525
526
527
528
529
 
# File 'app/models/completion_kit/run.rb', line 523

def broadcast_clear_responses
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_responses",
    html: '<tbody id="run_responses"></tbody>'
  )
end

#broadcast_progress ⇒ Object 



490
491
492
493
494
495
496
497
 
# File 'app/models/completion_kit/run.rb', line 490

def broadcast_progress
  reload
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_status_panel",
    html: render_engine_partial("completion_kit/runs/status_panel", run: self)
  )
end

#broadcast_response(response) ⇒ Object 



531
532
533
534
535
536
537
 
# File 'app/models/completion_kit/run.rb', line 531

def broadcast_response(response)
  broadcast_append_to(
    "completion_kit_run_#{id}",
    target: "run_responses",
    html: render_engine_partial("completion_kit/runs/response_row", run: self, response: response, index: responses.where("id <= ?", response.id).count)
  )
end

#broadcast_response_update(response) ⇒ Object 



539
540
541
542
543
544
545
 
# File 'app/models/completion_kit/run.rb', line 539

def broadcast_response_update(response)
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "response_#{response.id}",
    html: render_engine_partial("completion_kit/runs/response_row", run: self, response: response, index: responses.where("id <= ?", response.id).count)
  )
end

#broadcast_sort_toolbar ⇒ Object 



515
516
517
518
519
520
521
 
# File 'app/models/completion_kit/run.rb', line 515

def broadcast_sort_toolbar
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_sort_toolbar",
    html: render_engine_partial("completion_kit/runs/sort_toolbar", run: self)
  )
end

#broadcast_status_header ⇒ Object 



499
500
501
502
503
504
505
 
# File 'app/models/completion_kit/run.rb', line 499

def broadcast_status_header
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_status_header",
    html: render_engine_partial("completion_kit/runs/status_header", run: self)
  )
end

#broadcast_ui ⇒ Object 



483
484
485
486
487
488
 
# File 'app/models/completion_kit/run.rb', line 483

def broadcast_ui
  broadcast_progress
  broadcast_status_header
  broadcast_actions
  broadcast_sort_toolbar
end

#check_metrics ⇒ Object 



149
150
151
 
# File 'app/models/completion_kit/run.rb', line 149

def check_metrics
  metrics.where(metric_type: "check")
end

#generate_responses! ⇒ Object 



343
344
345
 
# File 'app/models/completion_kit/run.rb', line 343

def generate_responses!
  start!
end

#gradable? ⇒ Boolean

Returns:

  • (Boolean) 


157
158
159
 
# File 'app/models/completion_kit/run.rb', line 157

def gradable?
  llm_judge_configured? || check_metrics.any?
end

#gradable_metric_ids ⇒ Object 



116
117
118
119
120
 
# File 'app/models/completion_kit/run.rb', line 116

def gradable_metric_ids
  ids = check_metrics.pluck(:id)
  ids += llm_metrics.pluck(:id) if judge_model.present?
  ids
end

#judge_configured? ⇒ Boolean

Returns:

  • (Boolean) 


141
142
143
 
# File 'app/models/completion_kit/run.rb', line 141

def judge_configured?
  judge_model.present? && metrics.any? && ApiConfig.valid_for_model?(judge_model)
end

#judge_only? ⇒ Boolean

A scoring-only run grades a pre-existing column on the dataset instead of generating new outputs. No prompt is attached; the response text is read from row; no LLM generation happens.

Returns:

  • (Boolean) 


94
95
96
 
# File 'app/models/completion_kit/run.rb', line 94

def judge_only?
  prompt.nil?
end

#judge_only_input_data_checks? ⇒ Boolean

Returns:

  • (Boolean) 


161
162
163
164
165
166
167
168
 
# File 'app/models/completion_kit/run.rb', line 161

def judge_only_input_data_checks?
  return false unless judge_only?

  attached = run_metrics.filter_map(&:metric)
  return false if attached.empty?

  attached.all?(&:check?) && attached.all? { |m| m.check_config.to_h["target"] == "input_data" }
end

#llm_judge_configured? ⇒ Boolean

Returns:

  • (Boolean) 


153
154
155
 
# File 'app/models/completion_kit/run.rb', line 153

def llm_judge_configured?
  judge_model.present? && llm_metrics.any? && ApiConfig.valid_for_model?(judge_model)
end

#llm_metrics ⇒ Object 



145
146
147
 
# File 'app/models/completion_kit/run.rb', line 145

def llm_metrics
  metrics.where(metric_type: "llm_judge")
end

#mark_completed! ⇒ Object 



107
108
109
110
111
112
113
114
 
# File 'app/models/completion_kit/run.rb', line 107

def mark_completed!
  if all_responses_failed?
    fail_with_summary!(all_failed_summary)
  else
    update!(status: "completed")
    broadcast_ui
  end
end

#missing_dataset_variables ⇒ Object 



98
99
100
101
102
103
104
105
 
# File 'app/models/completion_kit/run.rb', line 98

def missing_dataset_variables
  return [] unless prompt
  vars = prompt.variables
  return [] if vars.empty?
  return vars if dataset.nil?

  vars - dataset.headers
end

#outstanding_work_zero? ⇒ Boolean

Returns:

  • (Boolean) 


122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'app/models/completion_kit/run.rb', line 122

def outstanding_work_zero?
  return false if responses.where.not(status: HasJobStatus::TERMINAL_STATUSES).exists?

  metric_ids = gradable_metric_ids
  return true if metric_ids.empty?

  succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
  expected_reviews = succeeded_response_ids.size * metric_ids.size
  return true if expected_reviews.zero?

  terminal_review_count = Review.where(
    response_id: succeeded_response_ids,
    metric_id: metric_ids,
    status: HasJobStatus::TERMINAL_STATUSES
  ).count

  terminal_review_count >= expected_reviews
end

#progress_snapshot ⇒ Object 



420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
 
# File 'app/models/completion_kit/run.rb', line 420

def progress_snapshot
  generated_done = responses.where(status: "succeeded").count
  generated_failed = responses.where(status: "failed").count
  generated_total = progress_total

  metric_ids = gradable_metric_ids
  metric_count = metric_ids.size
  judged_total = metric_count > 0 ? generated_done : 0
  judged_done = 0
  judged_failed = 0

  if metric_count > 0 && judged_total > 0
    succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
    review_counts = Review
      .where(response_id: succeeded_response_ids, metric_id: metric_ids)
      .group(:response_id, :status)
      .count
    succeeded_response_ids.each do |rid|
      ok = review_counts[[rid, "succeeded"]] || 0
      bad = review_counts[[rid, "failed"]] || 0
      next unless ok + bad == metric_count
      if bad > 0
        judged_failed += 1
      else
        judged_done += 1
      end
    end
  end

  {
    generated_done: generated_done,
    generated_total: generated_total,
    generated_failed: generated_failed,
    judged_done: judged_done,
    judged_total: judged_total,
    judged_failed: judged_failed
  }
end

#regrade! ⇒ Object 



347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
 
# File 'app/models/completion_kit/run.rb', line 347

def regrade!
  return false if metrics.empty? || !gradable?

  eligible_responses = responses.where(status: "succeeded")
  eligible_responses = eligible_responses.where.not(response_text: nil) unless judge_only_input_data_checks?
  response_ids = eligible_responses.pluck(:id)
  return false if response_ids.empty?

  transaction do
    Review.where(response_id: response_ids).update_all(
      status: "pending",
      attempts: 0,
      metric_version_id: nil,
      ai_score: nil,
      passed: nil,
      ai_feedback: nil,
      error_provider: nil,
      error_class: nil,
      error_status: nil,
      error_message: nil
    )
    update!(status: "running", failure_summary: nil, error_message: nil)

    response_ids.each do |rid|
      llm_metrics.each { |m| JudgeReviewJob.perform_later(rid, m.id, id) } if llm_judge_configured?
      check_metrics.each { |m| CheckReviewJob.perform_later(rid, m.id, id) }
    end
    RunCompletionCheckJob.perform_later(id)
  end

  broadcast_ui
  true
end

#replace_metrics!(metric_ids) ⇒ Object 



170
171
172
173
174
175
176
 
# File 'app/models/completion_kit/run.rb', line 170

def replace_metrics!(metric_ids)
  return unless metric_ids
  run_metrics.delete_all
  Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
    run_metrics.create!(metric_id: metric_id, position: index + 1)
  end
end

#rerun! ⇒ Object 



381
382
383
384
385
386
387
388
389
390
391
392
393
394
 
# File 'app/models/completion_kit/run.rb', line 381

def rerun!
  new_run = Run.create!(
    prompt_id: prompt_id,
    dataset_id: dataset_id,
    judge_model: judge_model,
    temperature: temperature,
    output_column: output_column,
    expected_column: expected_column,
    tag_names: tag_names,
    status: "pending"
  )
  new_run.replace_metrics!(metric_ids)
  new_run
end

#retry_failures!(only: nil) ⇒ Object 



396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
 
# File 'app/models/completion_kit/run.rb', line 396

def retry_failures!(only: nil)
  scope = responses.where(status: "failed")
  scope = scope.where(id: only) if only.present?

  transaction do
    failed_response_ids = scope.pluck(:id)
    Review.where(response_id: failed_response_ids, status: "failed").update_all(
      status: "pending",
      attempts: 0,
      error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
      ai_score: nil, passed: nil, ai_feedback: nil
    )
    scope.update_all(
      status: "pending",
      attempts: 0,
      error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
      response_text: nil
    )
    update!(status: "running")
    failed_response_ids.each { |rid| GenerateRowJob.perform_later(id, rid) }
  end
  self
end

#reviews_for_summary ⇒ Object 



178
179
180
181
182
183
184
185
 
# File 'app/models/completion_kit/run.rb', line 178

def reviews_for_summary
  @reviews_for_summary ||=
    if responses.loaded? && responses.all? { |response| response.association(:reviews).loaded? }
      responses.flat_map(&:reviews)
    else
      Review.where(response_id: responses.select(:id)).to_a
    end
end

#stale_review_summary ⇒ Object 



230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
 
# File 'app/models/completion_kit/run.rb', line 230

def stale_review_summary
  review_pairs = Review.where(response_id: response_ids)
                      .where.not(metric_id: nil)
                      .where.not(metric_version_id: nil)
                      .pluck(:metric_id, :metric_version_id, :metric_name)
  return {} if review_pairs.empty?

  metric_ids = review_pairs.map(&:first).uniq
  version_ids = review_pairs.map { |_, vid, _| vid }.uniq
  current_by_metric = MetricVersion.current.where(metric_id: metric_ids).pluck(:metric_id, :id, :version_number).each_with_object({}) do |(mid, vid, vnum), h|
    h[mid] = { id: vid, label: "v#{vnum}" }
  end
  label_by_version = MetricVersion.where(id: version_ids).pluck(:id, :version_number).each_with_object({}) { |(vid, vnum), h| h[vid] = "v#{vnum}" }

  summary = {}
  review_pairs.each do |metric_id, version_id, metric_name|
    current = current_by_metric[metric_id]
    next if current.nil?
    label = label_by_version[version_id]
    next if label.nil?
    next if label == current[:label]
    summary[metric_id] ||= { metric_name: metric_name, current_label: current[:label], stale_count: 0, scored_labels: [] }
    summary[metric_id][:stale_count] += 1
    summary[metric_id][:scored_labels] |= [label]
  end
  summary
end

#start! ⇒ Object 



258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
 
# File 'app/models/completion_kit/run.rb', line 258

def start!
  unless %w[pending failed].include?(status)
    return fail_with_summary!("Cannot start a run in state \"#{status}\". Use rerun to create a fresh copy, or retry_failures / regrade to work with the existing responses.")
  end

  rows = if dataset
           CsvProcessor.process_self(self)
         else
           [{}]
         end

  return fail_with_summary!("Dataset has no rows") if rows.empty?

  if judge_only?
    column = output_column.presence || "actual_output"
    unless judge_only_input_data_checks? || (dataset && dataset.headers.include?(column))
      return fail_with_summary!("Dataset has no \"#{column}\" column")
    end
  else
    client = LlmClient.for_model(prompt.llm_model, ApiConfig.for_model(prompt.llm_model))
    unless client.configured?
      return fail_with_summary!("LLM API not configured: #{client.configuration_errors.join(', ')}")
    end
  end

  begin
    transaction do
      responses.destroy_all
      update!(
        status: "running",
        progress_current: 0,
        progress_total: rows.length,
        failure_summary: nil,
        error_message: nil
      )

      now = Time.current
      out_col = output_column.presence || "actual_output"
      exp_col = expected_column.presence || "expected_output"
      judge = judge_only?
      has_output = judge && dataset && dataset.headers.include?(out_col)
      scope_defaults = Response.all.where_values_hash.symbolize_keys

      response_attrs = rows.each_with_index.map do |row, index|
        {
          run_id: id,
          status: judge ? "succeeded" : "pending",
          row_index: index,
          input_data: row.empty? ? nil : row.to_json,
          expected_output: row[exp_col],
          response_text: (judge && has_output ? row[out_col].to_s : nil),
          attempts: 0,
          created_at: now,
          updated_at: now
        }.merge(scope_defaults)
      end
      response_attrs.each_slice(INSERT_BATCH_SIZE) { |batch| Response.insert_all(batch) }
      responses.reset

      response_ids = responses.order(:row_index).pluck(:id)
      if judge
        judge_metrics = llm_judge_configured? ? llm_metrics.to_a : []
        chk_metrics = check_metrics.to_a
        review_jobs = response_ids.flat_map do |rid|
          judge_metrics.map { |m| JudgeReviewJob.new(rid, m.id, id) } +
            chk_metrics.map { |m| CheckReviewJob.new(rid, m.id, id) }
        end
        ActiveJob.perform_all_later(review_jobs) if review_jobs.any?
        RunCompletionCheckJob.perform_later(id)
      else
        ActiveJob.perform_all_later(response_ids.map { |rid| GenerateRowJob.new(id, rid) })
      end
    end
  rescue ActiveRecord::RecordInvalid => e
    reload
    return fail_with_summary!(e.record.errors.full_messages.to_sentence)
  end

  safely_broadcast do
    broadcast_ui
    broadcast_clear_responses
  end
  true
end