54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
|
# File 'app/jobs/completion_kit/judge_review_job.rb', line 54
def perform(response_id, metric_id, _run_id = nil)
@response_id = response_id
@metric_id = metric_id
response = Response.find(response_id)
metric = Metric.find(metric_id)
run = response.run
config = ApiConfig.for_model(run.judge_model).merge(judge_model: run.judge_model)
judge = JudgeService.new(config)
begin
evaluation = judge.evaluate(
response.response_text,
response.expected_output,
run.prompt&.template,
criteria: metric.instruction.to_s,
rubric_text: metric.display_rubric_text,
input_data: response.input_data,
human_examples: review_examples_for(metric, response)
)
rescue CompletionKit::ProviderError => e
record_terminal_failure!(e)
enqueue_completion_check
return
end
review = response.reviews.find_or_initialize_by(metric_id: metric.id)
current_metric_version = MetricVersion.ensure_current_for(metric)
review.assign_attributes(
metric_name: metric.name,
instruction: metric.instruction.to_s,
metric_version_id: current_metric_version.id,
status: "succeeded",
ai_score: evaluation[:score],
ai_feedback: evaluation[:feedback],
error_provider: nil, error_class: nil, error_status: nil, error_message: nil
)
review.save!
confirm_judging_capability(run.judge_model)
enqueue_completion_check
end
|