Class: CompletionKit::Run

Inherits:
ApplicationRecord show all
Includes:
Taggable, Turbo::Broadcastable
Defined in:
app/models/completion_kit/run.rb

Constant Summary collapse

STATUSES = 
%w[pending running completed failed].freeze
INSERT_BATCH_SIZE = 
1000
TOP_SCORE = 
5.0
CEILING_MEAN = 
4.8
CEILING_TOP_SHARE = 
0.9
CEILING_MIN_REVIEWS = 
10

Constants inherited from ApplicationRecord

ApplicationRecord::TenantScopedUniquenessValidator

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Taggable

#tag_names, #tag_names=

Instance Attribute Details

#avg_score ⇒ Object 



240
241
242
243
244
245
246
247
 
# File 'app/models/completion_kit/run.rb', line 240

def avg_score
  return @avg_score if defined?(@avg_score)

  scores = reviews_for_summary.map(&:ai_score).compact.map(&:to_f)
  return nil if scores.empty?

  (scores.sum / scores.length).round(2)
end

#check_pass_rate ⇒ Object 



270
271
272
273
274
275
276
277
278
 
# File 'app/models/completion_kit/run.rb', line 270

def check_pass_rate
  return @check_pass_rate if defined?(@check_pass_rate)

  resolved = reviews_for_summary.reject { |r| r.passed.nil? }
  return nil if resolved.empty?

  passed = resolved.count { |r| r.passed == true }
  (passed.to_f / resolved.length).round(2)
end

#metric_averages ⇒ Object 



249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
 
# File 'app/models/completion_kit/run.rb', line 249

def metric_averages
  return @metric_averages if defined?(@metric_averages)

  ceiling = self.class.low_score_ceiling
  reviews_for_summary.group_by(&:metric_name).filter_map do |name, reviews|
    scored = reviews.select { |r| r.ai_score.present? }
    if scored.any?
      scores = scored.map { |r| r.ai_score.to_f }
      { name: name, avg: (scores.sum / scores.length).round(1), count: scores.length,
        low_count: scores.count { |score| score < ceiling } }
    else
      resolved = reviews.reject { |r| r.passed.nil? }
      next if resolved.empty?

      passed = resolved.count { |r| r.passed == true }
      { name: name, kind: "check", pass_rate: (passed.to_f / resolved.length).round(2),
        count: resolved.length, low_count: resolved.length - passed }
    end
  end
end

#response_count ⇒ Object 



234
235
236
237
238
 
# File 'app/models/completion_kit/run.rb', line 234

def response_count
  return @response_count if defined?(@response_count)

  responses.size
end

Class Method Details

.display_scoped ⇒ Object 



32
33
34
35
 
# File 'app/models/completion_kit/run.rb', line 32

def self.display_scoped
  filter = CompletionKit.config.runs_display_scope
  filter ? all.instance_exec(&filter) : all
end

.low_score_ceiling ⇒ Object

Scores below this are the ones worth reading: low_count on each metric average counts them, so a caller can spot the dragging metric without pulling every review.



98
99
100
 
# File 'app/models/completion_kit/run.rb', line 98

def self.low_score_ceiling
  CompletionKit.config.medium_quality_threshold.to_f
end

.preload_summaries(runs) ⇒ Object

Batch-compute the list-view summaries (response count, avg score, check pass rate, per-metric averages) for a set of runs in a constant number of grouped queries, injecting the results so the index never loads a single response or review object. Mirrors the per-run reader methods exactly.



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'app/models/completion_kit/run.rb', line 45

def self.preload_summaries(runs)
  runs = runs.to_a
  return runs if runs.empty?

  run_ids = runs.map(&:id)
  counts = Response.where(run_id: run_ids).group(:run_id).count

  run_col = Arel.sql("completion_kit_responses.run_id")
  base = Review.joins(:response).where(completion_kit_responses: {run_id: run_ids})

  run_rows = base.group(run_col).pluck(
    run_col,
    Arel.sql("AVG(ai_score)"),
    Arel.sql("COUNT(passed)"),
    Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
  )
  run_stats = run_rows.each_with_object({}) do |(rid, avg, resolved, passed), h|
    h[rid] = {avg: avg, resolved: resolved.to_i, passed: passed.to_i}
  end

  metric_rows = base.group(run_col, :metric_name).pluck(
    run_col,
    :metric_name,
    Arel.sql("AVG(ai_score)"),
    Arel.sql("COUNT(ai_score)"),
    Arel.sql("SUM(CASE WHEN ai_score < #{low_score_ceiling} THEN 1 ELSE 0 END)"),
    Arel.sql("COUNT(passed)"),
    Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
  )
  metrics_by_run = metric_rows.group_by(&:first)

  runs.each do |run|
    run.response_count = counts.fetch(run.id, 0)

    stats = run_stats[run.id]
    run.avg_score = stats && stats[:avg] ? stats[:avg].to_f.round(2) : nil
    run.check_pass_rate = stats && stats[:resolved] > 0 ? (stats[:passed].to_f / stats[:resolved]).round(2) : nil

    run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, low, resolved, passed)|
      if scored.to_i > 0
        {name: name, avg: avg.to_f.round(1), count: scored.to_i, low_count: low.to_i}
      elsif resolved.to_i > 0
        {name: name, kind: "check", pass_rate: (passed.to_i.to_f / resolved.to_i).round(2),
         count: resolved.to_i, low_count: resolved.to_i - passed.to_i}
      end
    end
  end
  runs
end

.visible_run_ids ⇒ Object 



37
38
39
 
# File 'app/models/completion_kit/run.rb', line 37

def self.visible_run_ids
  display_scoped.select(:id)
end

Instance Method Details

#as_json(options = {}) ⇒ Object 



553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
 
# File 'app/models/completion_kit/run.rb', line 553

def as_json(options = {})
  snap = progress_snapshot
  {
    id: id, name: name, status: status, prompt_id: prompt_id,
    dataset_id: dataset_id, judge_model: judge_model, temperature: temperature,
    output_column: output_column,
    expected_column: expected_column,
    created_at: created_at, updated_at: updated_at,
    max_tokens: max_tokens, judge_temperature: judge_temperature,
    responses_count: responses.count, avg_score: avg_score,
    check_pass_rate: check_pass_rate,
    metric_averages: metric_averages,
    progress_current: snap[:generated_done],
    progress_total: snap[:generated_total],
    progress: {
      generated: { done: snap[:generated_done], total: snap[:generated_total], failed: snap[:generated_failed] },
      judged:    { done: snap[:judged_done],    total: snap[:judged_total],    failed: snap[:judged_failed] }
    },
    failed_response_ids: responses.where(status: "failed").pluck(:id),
    failure_summary: failure_summary,
    error_message: error_message,
    metric_ids: metric_ids,
    tags: tags.as_json
  }
end

#broadcast_actions ⇒ Object 



603
604
605
606
607
608
609
 
# File 'app/models/completion_kit/run.rb', line 603

def broadcast_actions
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_actions",
    html: render_engine_partial("completion_kit/runs/actions", run: self)
  )
end

#broadcast_clear_responses ⇒ Object 



619
620
621
622
623
624
625
 
# File 'app/models/completion_kit/run.rb', line 619

def broadcast_clear_responses
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_responses",
    html: '<tbody id="run_responses"></tbody>'
  )
end

#broadcast_progress ⇒ Object 



586
587
588
589
590
591
592
593
 
# File 'app/models/completion_kit/run.rb', line 586

def broadcast_progress
  reload
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_status_panel",
    html: render_engine_partial("completion_kit/runs/status_panel", run: self)
  )
end

#broadcast_response(response) ⇒ Object 



627
628
629
630
631
632
633
 
# File 'app/models/completion_kit/run.rb', line 627

def broadcast_response(response)
  broadcast_append_to(
    "completion_kit_run_#{id}",
    target: "run_responses",
    html: render_engine_partial("completion_kit/runs/response_row", run: self, response: response, index: responses.where("id <= ?", response.id).count)
  )
end

#broadcast_response_update(response) ⇒ Object 



635
636
637
638
639
640
641
 
# File 'app/models/completion_kit/run.rb', line 635

def broadcast_response_update(response)
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "response_#{response.id}",
    html: render_engine_partial("completion_kit/runs/response_row", run: self, response: response, index: responses.where("id <= ?", response.id).count)
  )
end

#broadcast_sort_toolbar ⇒ Object 



611
612
613
614
615
616
617
 
# File 'app/models/completion_kit/run.rb', line 611

def broadcast_sort_toolbar
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_sort_toolbar",
    html: render_engine_partial("completion_kit/runs/sort_toolbar", run: self)
  )
end

#broadcast_status_header ⇒ Object 



595
596
597
598
599
600
601
 
# File 'app/models/completion_kit/run.rb', line 595

def broadcast_status_header
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_status_header",
    html: render_engine_partial("completion_kit/runs/status_header", run: self)
  )
end

#broadcast_ui ⇒ Object 



579
580
581
582
583
584
 
# File 'app/models/completion_kit/run.rb', line 579

def broadcast_ui
  broadcast_progress
  broadcast_status_header
  broadcast_actions
  broadcast_sort_toolbar
end

#calibratable_metric ⇒ Object 



122
123
124
 
# File 'app/models/completion_kit/run.rb', line 122

def calibratable_metric
  llm_metrics.first
end

#check_metrics ⇒ Object 



196
197
198
 
# File 'app/models/completion_kit/run.rb', line 196

def check_metrics
  metrics.where(metric_type: "check")
end

#execute_start!(scope_defaults = {}) ⇒ Object 



357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
 
# File 'app/models/completion_kit/run.rb', line 357

def execute_start!(scope_defaults = {})
  rows = dataset ? CsvProcessor.process_self(self) : [{}]

  return fail_with_summary!("Dataset has no rows") if rows.empty?

  begin
    transaction do
      update!(progress_current: 0, progress_total: rows.length)

      now = Time.current
      out_col = output_column.presence || "actual_output"
      exp_col = expected_column.presence || "expected_output"
      judge = judge_only?
      has_output = judge && dataset && dataset.headers.include?(out_col)

      response_attrs = rows.each_with_index.map do |row, index|
        {
          run_id: id,
          status: judge ? "succeeded" : "pending",
          row_index: index,
          input_data: row.empty? ? nil : row.to_json,
          expected_output: row[exp_col],
          response_text: (judge && has_output ? row[out_col].to_s : nil),
          attempts: 0,
          created_at: now,
          updated_at: now
        }.merge(scope_defaults)
      end
      response_attrs.each_slice(INSERT_BATCH_SIZE) { |batch| Response.insert_all(batch) }
      responses.reset

      response_ids = responses.order(:row_index).pluck(:id)
      if judge
        judge_metrics = llm_judge_configured? ? llm_metrics.to_a : []
        chk_metrics = check_metrics.to_a
        review_jobs = response_ids.flat_map do |rid|
          judge_metrics.map { |m| JudgeReviewJob.new(rid, m.id, id) } +
            chk_metrics.map { |m| CheckReviewJob.new(rid, m.id, id) }
        end
        ActiveJob.perform_all_later(review_jobs) if review_jobs.any?
        RunCompletionCheckJob.perform_later(id)
      else
        ActiveJob.perform_all_later(response_ids.map { |rid| GenerateRowJob.new(id, rid) })
      end
    end
  rescue ActiveRecord::RecordInvalid => e
    reload
    return fail_with_summary!(e.record.errors.full_messages.to_sentence)
  end

  safely_broadcast do
    broadcast_ui
    broadcast_clear_responses
  end
  true
end

#fail_to_start!(message) ⇒ Object 



353
354
355
 
# File 'app/models/completion_kit/run.rb', line 353

def fail_to_start!(message)
  fail_with_summary!(message)
end

#generate_responses! ⇒ Object 



414
415
416
 
# File 'app/models/completion_kit/run.rb', line 414

def generate_responses!
  start!
end

#generation_options(prompt) ⇒ Object

The options every generation call for this run sends to the provider. max_tokens is omitted when unset so each client keeps its own default; setting it is how a run reproduces a production cap and stops the judge scoring truncated output.



456
457
458
459
460
 
# File 'app/models/completion_kit/run.rb', line 456

def generation_options(prompt)
  options = {model: prompt.llm_model, temperature: temperature}
  options[:max_tokens] = max_tokens if max_tokens
  options
end

#gradable? ⇒ Boolean

Returns:

  • (Boolean) 


204
205
206
 
# File 'app/models/completion_kit/run.rb', line 204

def gradable?
  llm_judge_configured? || check_metrics.any?
end

#gradable_metric_ids ⇒ Object 



163
164
165
166
167
 
# File 'app/models/completion_kit/run.rb', line 163

def gradable_metric_ids
  ids = check_metrics.pluck(:id)
  ids += llm_metrics.pluck(:id) if judge_model.present?
  ids
end

#judge_agreement ⇒ Object

How often a human agreed with the judge on this run's own responses. Scoped to the run rather than to a metric's current version, so the figure describes the scores actually shown on this page. Uses the same Wilson point as MetricAgreementStats so the two surfaces never disagree.



130
131
132
133
134
135
136
 
# File 'app/models/completion_kit/run.rb', line 130

def judge_agreement
  verdicts = Agreement.where(run_id: id).pluck(:verdict)
  return nil if verdicts.empty?

  point = AgreementMath.wilson_interval(successes: verdicts.count { |v| v == "agree" }, n: verdicts.length)[:point]
  { rate: point, sample_size: verdicts.length }
end

#judge_config ⇒ Object

Everything JudgeService needs to score this run. Judging defaults to temperature 0 so re-judging the same output yields the same score; anything above that makes the run's numbers irreproducible.



465
466
467
 
# File 'app/models/completion_kit/run.rb', line 465

def judge_config
  ApiConfig.for_model(judge_model).merge(judge_model: judge_model, judge_temperature: judge_temperature)
end

#judge_configured? ⇒ Boolean

Returns:

  • (Boolean) 


188
189
190
 
# File 'app/models/completion_kit/run.rb', line 188

def judge_configured?
  judge_model.present? && metrics.any? && ApiConfig.valid_for_model?(judge_model)
end

#judge_only? ⇒ Boolean

A scoring-only run grades a pre-existing column on the dataset instead of generating new outputs. No prompt is attached; the response text is read from row; no LLM generation happens.

Returns:

  • (Boolean) 


141
142
143
 
# File 'app/models/completion_kit/run.rb', line 141

def judge_only?
  prompt.nil?
end

#judge_only_input_data_checks? ⇒ Boolean

Returns:

  • (Boolean) 


208
209
210
211
212
213
214
215
 
# File 'app/models/completion_kit/run.rb', line 208

def judge_only_input_data_checks?
  return false unless judge_only?

  attached = run_metrics.filter_map(&:metric)
  return false if attached.empty?

  attached.all?(&:check?) && attached.all? { |m| m.check_config.to_h["target"] == "input_data" }
end

#llm_judge_configured? ⇒ Boolean

Returns:

  • (Boolean) 


200
201
202
 
# File 'app/models/completion_kit/run.rb', line 200

def llm_judge_configured?
  judge_model.present? && llm_metrics.any? && ApiConfig.valid_for_model?(judge_model)
end

#llm_metrics ⇒ Object 



192
193
194
 
# File 'app/models/completion_kit/run.rb', line 192

def llm_metrics
  metrics.where(metric_type: "llm_judge")
end

#mark_completed! ⇒ Object 



154
155
156
157
158
159
160
161
 
# File 'app/models/completion_kit/run.rb', line 154

def mark_completed!
  if all_responses_failed?
    fail_with_summary!(all_failed_summary)
  else
    update!(status: "completed")
    broadcast_ui
  end
end

#missing_dataset_variables ⇒ Object 



145
146
147
148
149
150
151
152
 
# File 'app/models/completion_kit/run.rb', line 145

def missing_dataset_variables
  return [] unless prompt
  vars = prompt.variables
  return [] if vars.empty?
  return vars if dataset.nil?

  vars - dataset.headers
end

#nondeterministic_judge? ⇒ Boolean

Returns:

  • (Boolean) 


469
470
471
 
# File 'app/models/completion_kit/run.rb', line 469

def nondeterministic_judge?
  judge_temperature.to_f > 0
end

#outstanding_work_zero? ⇒ Boolean

Returns:

  • (Boolean) 


169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'app/models/completion_kit/run.rb', line 169

def outstanding_work_zero?
  return false if responses.where.not(status: HasJobStatus::TERMINAL_STATUSES).exists?

  metric_ids = gradable_metric_ids
  return true if metric_ids.empty?

  succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
  expected_reviews = succeeded_response_ids.size * metric_ids.size
  return true if expected_reviews.zero?

  terminal_review_count = Review.where(
    response_id: succeeded_response_ids,
    metric_id: metric_ids,
    status: HasJobStatus::TERMINAL_STATUSES
  ).count

  terminal_review_count >= expected_reviews
end

#progress_snapshot ⇒ Object 



514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
 
# File 'app/models/completion_kit/run.rb', line 514

def progress_snapshot
  generated_done = responses.where(status: "succeeded").count
  generated_failed = responses.where(status: "failed").count
  generated_total = progress_total

  metric_ids = gradable_metric_ids
  metric_count = metric_ids.size
  judged_total = metric_count > 0 ? generated_done : 0
  judged_done = 0
  judged_failed = 0

  if metric_count > 0 && judged_total > 0
    succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
    review_counts = Review
      .where(response_id: succeeded_response_ids, metric_id: metric_ids)
      .group(:response_id, :status)
      .count
    succeeded_response_ids.each do |rid|
      ok = review_counts[[rid, "succeeded"]] || 0
      bad = review_counts[[rid, "failed"]] || 0
      next unless ok + bad == metric_count
      if bad > 0
        judged_failed += 1
      else
        judged_done += 1
      end
    end
  end

  {
    generated_done: generated_done,
    generated_total: generated_total,
    generated_failed: generated_failed,
    judged_done: judged_done,
    judged_total: judged_total,
    judged_failed: judged_failed
  }
end

#regrade! ⇒ Object 



418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
 
# File 'app/models/completion_kit/run.rb', line 418

def regrade!
  return false if metrics.empty? || !gradable?

  eligible_responses = responses.where(status: "succeeded")
  eligible_responses = eligible_responses.where.not(response_text: nil) unless judge_only_input_data_checks?
  response_ids = eligible_responses.pluck(:id)
  return false if response_ids.empty?

  transaction do
    Review.where(response_id: response_ids).update_all(
      status: "pending",
      attempts: 0,
      metric_version_id: nil,
      ai_score: nil,
      passed: nil,
      ai_feedback: nil,
      error_provider: nil,
      error_class: nil,
      error_status: nil,
      error_message: nil
    )
    update!(status: "running", failure_summary: nil, error_message: nil)

    response_ids.each do |rid|
      llm_metrics.each { |m| JudgeReviewJob.perform_later(rid, m.id, id) } if llm_judge_configured?
      check_metrics.each { |m| CheckReviewJob.perform_later(rid, m.id, id) }
    end
    RunCompletionCheckJob.perform_later(id)
  end

  broadcast_ui
  true
end

#replace_metrics!(metric_ids) ⇒ Object 



217
218
219
220
221
222
223
 
# File 'app/models/completion_kit/run.rb', line 217

def replace_metrics!(metric_ids)
  return unless metric_ids
  run_metrics.delete_all
  Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
    run_metrics.create!(metric_id: metric_id, position: index + 1)
  end
end

#rerun! ⇒ Object 



473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
 
# File 'app/models/completion_kit/run.rb', line 473

def rerun!
  new_run = Run.create!(
    prompt_id: prompt_id,
    dataset_id: dataset_id,
    judge_model: judge_model,
    temperature: temperature,
    max_tokens: max_tokens,
    judge_temperature: judge_temperature,
    output_column: output_column,
    expected_column: expected_column,
    tag_names: tag_names,
    status: "pending"
  )
  new_run.replace_metrics!(metric_ids)
  new_run
end

#retry_failures!(only: nil) ⇒ Object 



490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
 
# File 'app/models/completion_kit/run.rb', line 490

def retry_failures!(only: nil)
  scope = responses.where(status: "failed")
  scope = scope.where(id: only) if only.present?

  transaction do
    failed_response_ids = scope.pluck(:id)
    Review.where(response_id: failed_response_ids, status: "failed").update_all(
      status: "pending",
      attempts: 0,
      error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
      ai_score: nil, passed: nil, ai_feedback: nil
    )
    scope.update_all(
      status: "pending",
      attempts: 0,
      error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
      response_text: nil
    )
    update!(status: "running")
    failed_response_ids.each { |rid| GenerateRowJob.perform_later(id, rid) }
  end
  self
end

#reviews_for_summary ⇒ Object 



225
226
227
228
229
230
231
232
 
# File 'app/models/completion_kit/run.rb', line 225

def reviews_for_summary
  @reviews_for_summary ||=
    if responses.loaded? && responses.all? { |response| response.association(:reviews).loaded? }
      responses.flat_map(&:reviews)
    else
      Review.where(response_id: responses.select(:id)).to_a
    end
end

#scores_at_ceiling? ⇒ Boolean

A judge that scores almost everything at the top is usually failing to separate good output from bad, which reads as success to anyone who has not calibrated it. Detected either as a near-max mean or as nearly every score landing on the top band, and only once there are enough scores for the shape to mean anything.

Returns:

  • (Boolean) 


112
113
114
115
116
117
118
119
120
 
# File 'app/models/completion_kit/run.rb', line 112

def scores_at_ceiling?
  return false unless status == "completed"

  scores = reviews_for_summary.filter_map { |review| review.ai_score&.to_f }
  return false if scores.length < CEILING_MIN_REVIEWS
  return true if (scores.sum / scores.length) >= CEILING_MEAN

  (scores.count { |score| score >= TOP_SCORE }.to_f / scores.length) >= CEILING_TOP_SHARE
end

#stale_review_summary ⇒ Object 



280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
 
# File 'app/models/completion_kit/run.rb', line 280

def stale_review_summary
  review_pairs = Review.where(response_id: response_ids)
                      .where.not(metric_id: nil)
                      .where.not(metric_version_id: nil)
                      .pluck(:metric_id, :metric_version_id, :metric_name)
  return {} if review_pairs.empty?

  metric_ids = review_pairs.map(&:first).uniq
  version_ids = review_pairs.map { |_, vid, _| vid }.uniq
  current_by_metric = MetricVersion.current.where(metric_id: metric_ids).pluck(:metric_id, :id, :version_number).each_with_object({}) do |(mid, vid, vnum), h|
    h[mid] = { id: vid, label: "v#{vnum}" }
  end
  label_by_version = MetricVersion.where(id: version_ids).pluck(:id, :version_number).each_with_object({}) { |(vid, vnum), h| h[vid] = "v#{vnum}" }

  summary = {}
  review_pairs.each do |metric_id, version_id, metric_name|
    current = current_by_metric[metric_id]
    next if current.nil?
    label = label_by_version[version_id]
    next if label.nil?
    next if label == current[:label]
    summary[metric_id] ||= { metric_name: metric_name, current_label: current[:label], stale_count: 0, scored_labels: [] }
    summary[metric_id][:stale_count] += 1
    summary[metric_id][:scored_labels] |= [label]
  end
  summary
end

#start! ⇒ Object

Validates what can be checked cheaply, claims the run, and hands the expensive part (parsing the dataset, inserting a response per row, and enqueueing a job per row) to StartRunJob. Holding all that open inside the caller's request is what made runs_generate look like it had timed out when the run had in fact started.



313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
 
# File 'app/models/completion_kit/run.rb', line 313

def start!
  unless %w[pending failed].include?(status)
    return fail_with_summary!("Cannot start a run in state \"#{status}\". Use rerun to create a fresh copy, or retry_failures / regrade to work with the existing responses.")
  end

  return fail_with_summary!("Dataset has no rows") if dataset && dataset.row_count.zero?

  if judge_only?
    column = output_column.presence || "actual_output"
    unless judge_only_input_data_checks? || (dataset && dataset.headers.include?(column))
      return fail_with_summary!("Dataset has no \"#{column}\" column")
    end
  else
    client = LlmClient.for_model(prompt.llm_model, ApiConfig.for_model(prompt.llm_model))
    unless client.configured?
      return fail_with_summary!("LLM API not configured: #{client.configuration_errors.join(', ')}")
    end
  end

  begin
    transaction do
      responses.destroy_all
      update!(
        status: "running",
        progress_current: 0,
        progress_total: 0,
        failure_summary: nil,
        error_message: nil
      )
    end
  rescue ActiveRecord::RecordInvalid => e
    reload
    return fail_with_summary!(e.record.errors.full_messages.to_sentence)
  end

  StartRunJob.perform_later(id, Response.all.where_values_hash.symbolize_keys)
  safely_broadcast { broadcast_ui }
  true
end