Class: CompletionKit::JudgeService

Inherits:
Object
  • Object
show all
Defined in:
app/services/completion_kit/judge_service.rb

Instance Method Summary collapse

Constructor Details

#initialize(config = {}) ⇒ JudgeService

Returns a new instance of JudgeService.



5
6
7
8
9
 
# File 'app/services/completion_kit/judge_service.rb', line 5

def initialize(config = {})
  @config = config
  @judge_model = config[:judge_model] || CompletionKit.config.judge_model
  @judge_client = LlmClient.for_model(@judge_model, ApiConfig.for_model(@judge_model))
end

Instance Method Details

#evaluate(output, expected_output = nil, prompt = nil, criteria: nil, evaluation_steps: nil, rubric_text: nil, human_examples: nil, input_data: nil, **_extras) ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
 
# File 'app/services/completion_kit/judge_service.rb', line 11

def evaluate(output, expected_output = nil, prompt = nil, criteria: nil, evaluation_steps: nil, rubric_text: nil, human_examples: nil, input_data: nil, **_extras)
  return { score: 1, feedback: "Judge not configured" } unless @judge_client.configured?

  judge_prompt = build_judge_prompt(output, expected_output, prompt,
    criteria: criteria, evaluation_steps: evaluation_steps,
    rubric_text: rubric_text, human_examples: human_examples,
    input_data: input_data)

  response = @judge_client.generate_completion(judge_prompt, model: @judge_model)
  raise StandardError, response if response.start_with?("Error:")
  parse_judge_response(response)
rescue Faraday::Error
  raise
rescue => e
  { score: 1, feedback: "Error during evaluation: #{e.message}" }
end