Module: CompletionKit::JudgeCalibrationExamples

Defined in:
app/services/completion_kit/judge_variant_generator.rb

Class Method Summary collapse

Class Method Details

.for(metric, limit: 8) ⇒ Object 



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'app/services/completion_kit/judge_variant_generator.rb', line 90

def for(metric, limit: 8)
  disagreements = Calibration.where(metric_id: metric.id, verdict: "disagree")
                             .includes(response: :reviews)
                             .order(created_at: :desc)
                             .limit(limit)
  disagreements.map do |cal|
    review = cal.response.reviews.find { |r| r.metric_id == metric.id }
    {
      input: cal.response.input_data,
      output: cal.response.response_text,
      judge_score: review&.ai_score,
      judge_feedback: review&.ai_feedback,
      human_score: cal.corrected_score,
      human_note: cal.note
    }
  end
end