Class: CompletionKit::Run

Inherits:
ApplicationRecord show all
Includes:
Taggable, Turbo::Broadcastable
Defined in:
app/models/completion_kit/run.rb

Constant Summary collapse

STATUSES = 
%w[pending running completed failed].freeze
INSERT_BATCH_SIZE = 
1000
REVIEW_RETRY_RESET = 
{
  status: "pending",
  attempts: 0,
  error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
  ai_score: nil, passed: nil, ai_feedback: nil
}.freeze
TOP_SCORE = 
5.0
CEILING_MEAN = 
4.8
CEILING_TOP_SHARE = 
0.9
CEILING_MIN_REVIEWS = 
10

Constants inherited from ApplicationRecord

ApplicationRecord::TenantScopedUniquenessValidator

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Taggable

#tag_names, #tag_names=

Instance Attribute Details

#avg_score ⇒ Object 



246
247
248
249
250
251
252
253
 
# File 'app/models/completion_kit/run.rb', line 246

def avg_score
  return @avg_score if defined?(@avg_score)

  scores = reviews_for_summary.map(&:ai_score).compact.map(&:to_f)
  return nil if scores.empty?

  (scores.sum / scores.length).round(2)
end

#check_pass_rate ⇒ Object 



276
277
278
279
280
281
282
283
284
 
# File 'app/models/completion_kit/run.rb', line 276

def check_pass_rate
  return @check_pass_rate if defined?(@check_pass_rate)

  resolved = reviews_for_summary.reject { |r| r.passed.nil? }
  return nil if resolved.empty?

  passed = resolved.count { |r| r.passed == true }
  (passed.to_f / resolved.length).round(2)
end

#metric_averages ⇒ Object 



255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
 
# File 'app/models/completion_kit/run.rb', line 255

def metric_averages
  return @metric_averages if defined?(@metric_averages)

  ceiling = self.class.low_score_ceiling
  reviews_for_summary.group_by(&:metric_name).filter_map do |name, reviews|
    scored = reviews.select { |r| r.ai_score.present? }
    if scored.any?
      scores = scored.map { |r| r.ai_score.to_f }
      { name: name, avg: (scores.sum / scores.length).round(1), count: scores.length,
        low_count: scores.count { |score| score < ceiling } }
    else
      resolved = reviews.reject { |r| r.passed.nil? }
      next if resolved.empty?

      passed = resolved.count { |r| r.passed == true }
      { name: name, kind: "check", pass_rate: (passed.to_f / resolved.length).round(2),
        count: resolved.length, low_count: resolved.length - passed }
    end
  end
end

#response_count ⇒ Object 



240
241
242
243
244
 
# File 'app/models/completion_kit/run.rb', line 240

def response_count
  return @response_count if defined?(@response_count)

  responses.size
end

Class Method Details

.display_scoped ⇒ Object 



38
39
40
41
 
# File 'app/models/completion_kit/run.rb', line 38

def self.display_scoped
  filter = CompletionKit.config.runs_display_scope
  filter ? all.instance_exec(&filter) : all
end

.low_score_ceiling ⇒ Object

Scores below this are the ones worth reading: low_count on each metric average counts them, so a caller can spot the dragging metric without pulling every review.



104
105
106
 
# File 'app/models/completion_kit/run.rb', line 104

def self.low_score_ceiling
  CompletionKit.config.medium_quality_threshold.to_f
end

.preload_summaries(runs) ⇒ Object

Batch-compute the list-view summaries (response count, avg score, check pass rate, per-metric averages) for a set of runs in a constant number of grouped queries, injecting the results so the index never loads a single response or review object. Mirrors the per-run reader methods exactly.



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'app/models/completion_kit/run.rb', line 51

def self.preload_summaries(runs)
  runs = runs.to_a
  return runs if runs.empty?

  run_ids = runs.map(&:id)
  counts = Response.where(run_id: run_ids).group(:run_id).count

  run_col = Arel.sql("completion_kit_responses.run_id")
  base = Review.joins(:response).where(completion_kit_responses: {run_id: run_ids})

  run_rows = base.group(run_col).pluck(
    run_col,
    Arel.sql("AVG(ai_score)"),
    Arel.sql("COUNT(passed)"),
    Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
  )
  run_stats = run_rows.each_with_object({}) do |(rid, avg, resolved, passed), h|
    h[rid] = {avg: avg, resolved: resolved.to_i, passed: passed.to_i}
  end

  metric_rows = base.group(run_col, :metric_name).pluck(
    run_col,
    :metric_name,
    Arel.sql("AVG(ai_score)"),
    Arel.sql("COUNT(ai_score)"),
    Arel.sql("SUM(CASE WHEN ai_score < #{low_score_ceiling} THEN 1 ELSE 0 END)"),
    Arel.sql("COUNT(passed)"),
    Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
  )
  metrics_by_run = metric_rows.group_by(&:first)

  runs.each do |run|
    run.response_count = counts.fetch(run.id, 0)

    stats = run_stats[run.id]
    run.avg_score = stats && stats[:avg] ? stats[:avg].to_f.round(2) : nil
    run.check_pass_rate = stats && stats[:resolved] > 0 ? (stats[:passed].to_f / stats[:resolved]).round(2) : nil

    run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, low, resolved, passed)|
      if scored.to_i > 0
        {name: name, avg: avg.to_f.round(1), count: scored.to_i, low_count: low.to_i}
      elsif resolved.to_i > 0
        {name: name, kind: "check", pass_rate: (passed.to_i.to_f / resolved.to_i).round(2),
         count: resolved.to_i, low_count: resolved.to_i - passed.to_i}
      end
    end
  end
  runs
end

.visible_run_ids ⇒ Object 



43
44
45
 
# File 'app/models/completion_kit/run.rb', line 43

def self.visible_run_ids
  display_scoped.select(:id)
end

Instance Method Details

#as_json(options = {}) ⇒ Object 



582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
 
# File 'app/models/completion_kit/run.rb', line 582

def as_json(options = {})
  snap = progress_snapshot
  {
    id: id, name: name, status: status, prompt_id: prompt_id,
    dataset_id: dataset_id, judge_model: judge_model, temperature: temperature,
    output_column: output_column,
    expected_column: expected_column,
    created_at: created_at, updated_at: updated_at,
    max_tokens: max_tokens, judge_temperature: judge_temperature,
    temperature_ignored: temperature_ignored, judge_temperature_ignored: judge_temperature_ignored,
    responses_count: responses.count, avg_score: avg_score,
    check_pass_rate: check_pass_rate,
    metric_averages: metric_averages,
    progress_current: snap[:generated_done],
    progress_total: snap[:generated_total],
    progress: {
      generated: { done: snap[:generated_done], total: snap[:generated_total], failed: snap[:generated_failed] },
      judged:    { done: snap[:judged_done],    total: snap[:judged_total],    failed: snap[:judged_failed] }
    },
    failed_response_ids: responses.where(status: "failed").pluck(:id),
    failure_summary: failure_summary,
    error_message: error_message,
    metric_ids: metric_ids,
    tags: tags.as_json
  }
end

#broadcast_actions ⇒ Object 



633
634
635
636
637
638
639
 
# File 'app/models/completion_kit/run.rb', line 633

def broadcast_actions
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_actions",
    html: render_engine_partial("completion_kit/runs/actions", run: self)
  )
end

#broadcast_clear_responses ⇒ Object 



649
650
651
652
653
654
655
 
# File 'app/models/completion_kit/run.rb', line 649

def broadcast_clear_responses
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_responses",
    html: '<tbody id="run_responses"></tbody>'
  )
end

#broadcast_progress ⇒ Object 



616
617
618
619
620
621
622
623
 
# File 'app/models/completion_kit/run.rb', line 616

def broadcast_progress
  reload
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_status_panel",
    html: render_engine_partial("completion_kit/runs/status_panel", run: self)
  )
end

#broadcast_response(response) ⇒ Object 



657
658
659
660
661
662
663
 
# File 'app/models/completion_kit/run.rb', line 657

def broadcast_response(response)
  broadcast_append_to(
    "completion_kit_run_#{id}",
    target: "run_responses",
    html: render_engine_partial("completion_kit/runs/response_row", run: self, response: response, index: responses.where("id <= ?", response.id).count)
  )
end

#broadcast_response_update(response) ⇒ Object 



665
666
667
668
669
670
671
 
# File 'app/models/completion_kit/run.rb', line 665

def broadcast_response_update(response)
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "response_#{response.id}",
    html: render_engine_partial("completion_kit/runs/response_row", run: self, response: response, index: responses.where("id <= ?", response.id).count)
  )
end

#broadcast_sort_toolbar ⇒ Object 



641
642
643
644
645
646
647
 
# File 'app/models/completion_kit/run.rb', line 641

def broadcast_sort_toolbar
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_sort_toolbar",
    html: render_engine_partial("completion_kit/runs/sort_toolbar", run: self)
  )
end

#broadcast_status_header ⇒ Object 



625
626
627
628
629
630
631
 
# File 'app/models/completion_kit/run.rb', line 625

def broadcast_status_header
  broadcast_replace_to(
    "completion_kit_run_#{id}",
    target: "run_status_header",
    html: render_engine_partial("completion_kit/runs/status_header", run: self)
  )
end

#broadcast_ui ⇒ Object 



609
610
611
612
613
614
 
# File 'app/models/completion_kit/run.rb', line 609

def broadcast_ui
  broadcast_progress
  broadcast_status_header
  broadcast_actions
  broadcast_sort_toolbar
end

#calibratable_metric ⇒ Object 



128
129
130
 
# File 'app/models/completion_kit/run.rb', line 128

def calibratable_metric
  llm_metrics.first
end

#check_metrics ⇒ Object 



202
203
204
 
# File 'app/models/completion_kit/run.rb', line 202

def check_metrics
  metrics.where(metric_type: "check")
end

#enqueue_review_retries(pairs) ⇒ Object 



529
530
531
532
533
534
535
536
537
538
539
540
541
 
# File 'app/models/completion_kit/run.rb', line 529

def enqueue_review_retries(pairs)
  return if pairs.empty?

  kinds = Metric.where(id: pairs.map(&:last).uniq).pluck(:id, :metric_type).to_h
  pairs.each do |response_id, metric_id|
    if kinds[metric_id] == "check"
      CheckReviewJob.perform_later(response_id, metric_id, id)
    else
      JudgeReviewJob.perform_later(response_id, metric_id, id)
    end
  end
  RunCompletionCheckJob.perform_later(id)
end

#execute_start!(scope_defaults = {}) ⇒ Object 



365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
 
# File 'app/models/completion_kit/run.rb', line 365

def execute_start!(scope_defaults = {})
  rows = dataset ? CsvProcessor.process_self(self) : [{}]

  return fail_with_summary!("Dataset has no rows") if rows.empty?

  begin
    transaction do
      update!(progress_current: 0, progress_total: rows.length)

      now = Time.current
      out_col = output_column.presence || "actual_output"
      exp_col = expected_column.presence || "expected_output"
      judge = judge_only?
      has_output = judge && dataset && dataset.headers.include?(out_col)

      response_attrs = rows.each_with_index.map do |row, index|
        {
          run_id: id,
          status: judge ? "succeeded" : "pending",
          row_index: index,
          input_data: row.empty? ? nil : row.to_json,
          expected_output: row[exp_col],
          response_text: (judge && has_output ? row[out_col].to_s : nil),
          attempts: 0,
          created_at: now,
          updated_at: now
        }.merge(scope_defaults)
      end
      response_attrs.each_slice(INSERT_BATCH_SIZE) { |batch| Response.insert_all(batch) }
      responses.reset

      response_ids = responses.order(:row_index).pluck(:id)
      if judge
        judge_metrics = llm_judge_configured? ? llm_metrics.to_a : []
        chk_metrics = check_metrics.to_a
        review_jobs = response_ids.flat_map do |rid|
          judge_metrics.map { |m| JudgeReviewJob.new(rid, m.id, id) } +
            chk_metrics.map { |m| CheckReviewJob.new(rid, m.id, id) }
        end
        ActiveJob.perform_all_later(review_jobs) if review_jobs.any?
        RunCompletionCheckJob.perform_later(id)
      else
        ActiveJob.perform_all_later(response_ids.map { |rid| GenerateRowJob.new(id, rid) })
      end
    end
  rescue ActiveRecord::RecordInvalid => e
    reload
    return fail_with_summary!(e.record.errors.full_messages.to_sentence)
  end

  safely_broadcast do
    broadcast_ui
    broadcast_clear_responses
  end
  true
end

#fail_to_start!(message) ⇒ Object 



361
362
363
 
# File 'app/models/completion_kit/run.rb', line 361

def fail_to_start!(message)
  fail_with_summary!(message)
end

#generate_responses! ⇒ Object 



422
423
424
 
# File 'app/models/completion_kit/run.rb', line 422

def generate_responses!
  start!
end

#generation_options(prompt) ⇒ Object

The options every generation call for this run sends to the provider. max_tokens is omitted when unset so each client keeps its own default; setting it is how a run reproduces a production cap and stops the judge scoring truncated output.



464
465
466
467
468
 
# File 'app/models/completion_kit/run.rb', line 464

def generation_options(prompt)
  options = {model: prompt.llm_model, temperature: temperature}
  options[:max_tokens] = max_tokens if max_tokens
  options
end

#gradable? ⇒ Boolean

Returns:

  • (Boolean) 


210
211
212
 
# File 'app/models/completion_kit/run.rb', line 210

def gradable?
  llm_judge_configured? || check_metrics.any?
end

#gradable_metric_ids ⇒ Object 



169
170
171
172
173
 
# File 'app/models/completion_kit/run.rb', line 169

def gradable_metric_ids
  ids = check_metrics.pluck(:id)
  ids += llm_metrics.pluck(:id) if judge_model.present?
  ids
end

#judge_agreement ⇒ Object

How often a human agreed with the judge on this run's own responses. Scoped to the run rather than to a metric's current version, so the figure describes the scores actually shown on this page. Uses the same Wilson point as MetricAgreementStats so the two surfaces never disagree.



136
137
138
139
140
141
142
 
# File 'app/models/completion_kit/run.rb', line 136

def judge_agreement
  verdicts = Agreement.where(run_id: id).pluck(:verdict)
  return nil if verdicts.empty?

  point = AgreementMath.wilson_interval(successes: verdicts.count { |v| v == "agree" }, n: verdicts.length)[:point]
  { rate: point, sample_size: verdicts.length }
end

#judge_config ⇒ Object

Everything JudgeService needs to score this run. Judging defaults to temperature 0 so re-judging the same output yields the same score; anything above that makes the run's numbers irreproducible.



473
474
475
 
# File 'app/models/completion_kit/run.rb', line 473

def judge_config
  ApiConfig.for_model(judge_model).merge(judge_model: judge_model, judge_temperature: judge_temperature)
end

#judge_configured? ⇒ Boolean

Returns:

  • (Boolean) 


194
195
196
 
# File 'app/models/completion_kit/run.rb', line 194

def judge_configured?
  judge_model.present? && metrics.any? && ApiConfig.valid_for_model?(judge_model)
end

#judge_only? ⇒ Boolean

A scoring-only run grades a pre-existing column on the dataset instead of generating new outputs. No prompt is attached; the response text is read from row; no LLM generation happens.

Returns:

  • (Boolean) 


147
148
149
 
# File 'app/models/completion_kit/run.rb', line 147

def judge_only?
  prompt.nil?
end

#judge_only_input_data_checks? ⇒ Boolean

Returns:

  • (Boolean) 


214
215
216
217
218
219
220
221
 
# File 'app/models/completion_kit/run.rb', line 214

def judge_only_input_data_checks?
  return false unless judge_only?

  attached = run_metrics.filter_map(&:metric)
  return false if attached.empty?

  attached.all?(&:check?) && attached.all? { |m| m.check_config.to_h["target"] == "input_data" }
end

#llm_judge_configured? ⇒ Boolean

Returns:

  • (Boolean) 


206
207
208
 
# File 'app/models/completion_kit/run.rb', line 206

def llm_judge_configured?
  judge_model.present? && llm_metrics.any? && ApiConfig.valid_for_model?(judge_model)
end

#llm_metrics ⇒ Object 



198
199
200
 
# File 'app/models/completion_kit/run.rb', line 198

def llm_metrics
  metrics.where(metric_type: "llm_judge")
end

#mark_completed! ⇒ Object 



160
161
162
163
164
165
166
167
 
# File 'app/models/completion_kit/run.rb', line 160

def mark_completed!
  if all_responses_failed?
    fail_with_summary!(all_failed_summary)
  else
    update!(status: "completed")
    broadcast_ui
  end
end

#missing_dataset_variables ⇒ Object 



151
152
153
154
155
156
157
158
 
# File 'app/models/completion_kit/run.rb', line 151

def missing_dataset_variables
  return [] unless prompt
  vars = prompt.variables
  return [] if vars.empty?
  return vars if dataset.nil?

  vars - dataset.headers
end

#nondeterministic_judge? ⇒ Boolean

Returns:

  • (Boolean) 


477
478
479
 
# File 'app/models/completion_kit/run.rb', line 477

def nondeterministic_judge?
  judge_temperature.to_f > 0 || judge_temperature_ignored?
end

#outstanding_work_zero? ⇒ Boolean

Returns:

  • (Boolean) 


175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'app/models/completion_kit/run.rb', line 175

def outstanding_work_zero?
  return false if responses.where.not(status: HasJobStatus::TERMINAL_STATUSES).exists?

  metric_ids = gradable_metric_ids
  return true if metric_ids.empty?

  succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
  expected_reviews = succeeded_response_ids.size * metric_ids.size
  return true if expected_reviews.zero?

  terminal_review_count = Review.where(
    response_id: succeeded_response_ids,
    metric_id: metric_ids,
    status: HasJobStatus::TERMINAL_STATUSES
  ).count

  terminal_review_count >= expected_reviews
end

#progress_snapshot ⇒ Object 



543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
 
# File 'app/models/completion_kit/run.rb', line 543

def progress_snapshot
  generated_done = responses.where(status: "succeeded").count
  generated_failed = responses.where(status: "failed").count
  generated_total = progress_total

  metric_ids = gradable_metric_ids
  metric_count = metric_ids.size
  judged_total = metric_count > 0 ? generated_done : 0
  judged_done = 0
  judged_failed = 0

  if metric_count > 0 && judged_total > 0
    succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
    review_counts = Review
      .where(response_id: succeeded_response_ids, metric_id: metric_ids)
      .group(:response_id, :status)
      .count
    succeeded_response_ids.each do |rid|
      ok = review_counts[[rid, "succeeded"]] || 0
      bad = review_counts[[rid, "failed"]] || 0
      next unless ok + bad == metric_count
      if bad > 0
        judged_failed += 1
      else
        judged_done += 1
      end
    end
  end

  {
    generated_done: generated_done,
    generated_total: generated_total,
    generated_failed: generated_failed,
    judged_done: judged_done,
    judged_total: judged_total,
    judged_failed: judged_failed
  }
end

#regrade! ⇒ Object 



426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
 
# File 'app/models/completion_kit/run.rb', line 426

def regrade!
  return false if metrics.empty? || !gradable?

  eligible_responses = responses.where(status: "succeeded")
  eligible_responses = eligible_responses.where.not(response_text: nil) unless judge_only_input_data_checks?
  response_ids = eligible_responses.pluck(:id)
  return false if response_ids.empty?

  transaction do
    Review.where(response_id: response_ids).update_all(
      status: "pending",
      attempts: 0,
      metric_version_id: nil,
      ai_score: nil,
      passed: nil,
      ai_feedback: nil,
      error_provider: nil,
      error_class: nil,
      error_status: nil,
      error_message: nil
    )
    update!(status: "running", failure_summary: nil, error_message: nil, judge_temperature_ignored: false)

    response_ids.each do |rid|
      llm_metrics.each { |m| JudgeReviewJob.perform_later(rid, m.id, id) } if llm_judge_configured?
      check_metrics.each { |m| CheckReviewJob.perform_later(rid, m.id, id) }
    end
    RunCompletionCheckJob.perform_later(id)
  end

  broadcast_ui
  true
end

#replace_metrics!(metric_ids) ⇒ Object 



223
224
225
226
227
228
229
 
# File 'app/models/completion_kit/run.rb', line 223

def replace_metrics!(metric_ids)
  return unless metric_ids
  run_metrics.delete_all
  Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
    run_metrics.create!(metric_id: metric_id, position: index + 1)
  end
end

#rerun! ⇒ Object 



481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
 
# File 'app/models/completion_kit/run.rb', line 481

def rerun!
  new_run = Run.create!(
    prompt_id: prompt_id,
    dataset_id: dataset_id,
    judge_model: judge_model,
    temperature: temperature,
    max_tokens: max_tokens,
    judge_temperature: judge_temperature,
    output_column: output_column,
    expected_column: expected_column,
    tag_names: tag_names,
    status: "pending"
  )
  new_run.replace_metrics!(metric_ids)
  new_run
end

#retry_failures!(only: nil) ⇒ Object 



498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
 
# File 'app/models/completion_kit/run.rb', line 498

def retry_failures!(only: nil)
  scope = responses.where(status: "failed")
  scope = scope.where(id: only) if only.present?
  failed_response_ids = scope.pluck(:id)

  # A response can generate fine and still have a review blow up, so those
  # reviews are retryable on their own without regenerating the response.
  unscored = Review.where(status: "failed")
                   .where.not(metric_id: nil)
                   .where(response_id: responses.where(status: "succeeded").select(:id))
  unscored = unscored.where(response_id: only) if only.present?
  unscored_pairs = unscored.pluck(:response_id, :metric_id)

  return self if failed_response_ids.empty? && unscored_pairs.empty?

  transaction do
    Review.where(response_id: failed_response_ids, status: "failed").update_all(REVIEW_RETRY_RESET)
    unscored.update_all(REVIEW_RETRY_RESET)
    scope.update_all(
      status: "pending",
      attempts: 0,
      error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
      response_text: nil
    )
    update!(status: "running")
    failed_response_ids.each { |rid| GenerateRowJob.perform_later(id, rid) }
    enqueue_review_retries(unscored_pairs)
  end
  self
end

#reviews_for_summary ⇒ Object 



231
232
233
234
235
236
237
238
 
# File 'app/models/completion_kit/run.rb', line 231

def reviews_for_summary
  @reviews_for_summary ||=
    if responses.loaded? && responses.all? { |response| response.association(:reviews).loaded? }
      responses.flat_map(&:reviews)
    else
      Review.where(response_id: responses.select(:id)).to_a
    end
end

#scores_at_ceiling? ⇒ Boolean

A judge that scores almost everything at the top is usually failing to separate good output from bad, which reads as success to anyone who has not calibrated it. Detected either as a near-max mean or as nearly every score landing on the top band, and only once there are enough scores for the shape to mean anything.

Returns:

  • (Boolean) 


118
119
120
121
122
123
124
125
126
 
# File 'app/models/completion_kit/run.rb', line 118

def scores_at_ceiling?
  return false unless status == "completed"

  scores = reviews_for_summary.filter_map { |review| review.ai_score&.to_f }
  return false if scores.length < CEILING_MIN_REVIEWS
  return true if (scores.sum / scores.length) >= CEILING_MEAN

  (scores.count { |score| score >= TOP_SCORE }.to_f / scores.length) >= CEILING_TOP_SHARE
end

#stale_review_summary ⇒ Object 



286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
 
# File 'app/models/completion_kit/run.rb', line 286

def stale_review_summary
  review_pairs = Review.where(response_id: response_ids)
                      .where.not(metric_id: nil)
                      .where.not(metric_version_id: nil)
                      .pluck(:metric_id, :metric_version_id, :metric_name)
  return {} if review_pairs.empty?

  metric_ids = review_pairs.map(&:first).uniq
  version_ids = review_pairs.map { |_, vid, _| vid }.uniq
  current_by_metric = MetricVersion.current.where(metric_id: metric_ids).pluck(:metric_id, :id, :version_number).each_with_object({}) do |(mid, vid, vnum), h|
    h[mid] = { id: vid, label: "v#{vnum}" }
  end
  label_by_version = MetricVersion.where(id: version_ids).pluck(:id, :version_number).each_with_object({}) { |(vid, vnum), h| h[vid] = "v#{vnum}" }

  summary = {}
  review_pairs.each do |metric_id, version_id, metric_name|
    current = current_by_metric[metric_id]
    next if current.nil?
    label = label_by_version[version_id]
    next if label.nil?
    next if label == current[:label]
    summary[metric_id] ||= { metric_name: metric_name, current_label: current[:label], stale_count: 0, scored_labels: [] }
    summary[metric_id][:stale_count] += 1
    summary[metric_id][:scored_labels] |= [label]
  end
  summary
end

#start! ⇒ Object

Validates what can be checked cheaply, claims the run, and hands the expensive part (parsing the dataset, inserting a response per row, and enqueueing a job per row) to StartRunJob. Holding all that open inside the caller's request is what made runs_generate look like it had timed out when the run had in fact started.



319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
 
# File 'app/models/completion_kit/run.rb', line 319

def start!
  unless %w[pending failed].include?(status)
    return fail_with_summary!("Cannot start a run in state \"#{status}\". Use rerun to create a fresh copy, or retry_failures / regrade to work with the existing responses.")
  end

  return fail_with_summary!("Dataset has no rows") if dataset && dataset.row_count.zero?

  if judge_only?
    column = output_column.presence || "actual_output"
    unless judge_only_input_data_checks? || (dataset && dataset.headers.include?(column))
      return fail_with_summary!("Dataset has no \"#{column}\" column")
    end
  else
    client = LlmClient.for_model(prompt.llm_model, ApiConfig.for_model(prompt.llm_model))
    unless client.configured?
      return fail_with_summary!("LLM API not configured: #{client.configuration_errors.join(', ')}")
    end
  end

  begin
    transaction do
      responses.destroy_all
      update!(
        status: "running",
        progress_current: 0,
        progress_total: 0,
        failure_summary: nil,
        error_message: nil,
        temperature_ignored: false,
        judge_temperature_ignored: false
      )
    end
  rescue ActiveRecord::RecordInvalid => e
    reload
    return fail_with_summary!(e.record.errors.full_messages.to_sentence)
  end

  StartRunJob.perform_later(id, Response.all.where_values_hash.symbolize_keys)
  safely_broadcast { broadcast_ui }
  true
end