Class: RubricLLM::Judge

Inherits:
Object
  • Object
show all
Defined in:
lib/rubric_llm/judge.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config:) ⇒ Judge

Returns a new instance of Judge.



9
10
11
 
# File 'lib/rubric_llm/judge.rb', line 9

def initialize(config:)
  @config = config
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



7
8
9
 
# File 'lib/rubric_llm/judge.rb', line 7

def config
  @config
end

Instance Method Details

#call(system_prompt:, user_prompt:) ⇒ Object

Send a prompt to the judge LLM and return the parsed JSON response. Retries transient failures with exponential backoff.



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/rubric_llm/judge.rb', line 15

def call(system_prompt:, user_prompt:)
  config.validate!
  attempts = 0
  begin
    attempts += 1
    chat = RubyLLM.chat(model: config.judge_model, provider: config.judge_provider)
    chat.with_temperature(config.temperature)
    chat.with_params(max_tokens: config.max_tokens)

    full_system_prompt = build_system_prompt(system_prompt)
    chat.with_instructions(full_system_prompt)
    response = chat.ask(user_prompt)
    parse_json(response.content)
  rescue StandardError => e
    raise JudgeError, "Judge call failed: #{e.message}" if attempts > config.max_retries

    sleep(config.retry_base_delay * (2**(attempts - 1)))
    retry
  end
end

#parse_json(text) ⇒ Object

Parse JSON from LLM output with multiple strategies:

  1. Direct JSON.parse

  2. Extract from markdown code fence

  3. Return nil (never raises)



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/rubric_llm/judge.rb', line 40

def parse_json(text)
  return nil if text.nil? || text.strip.empty?

  # Try direct parse
  JSON.parse(text)
rescue JSON::ParserError
  # Try extracting from code fence
  if (match = text.match(/```(?:json)?\s*\n?(.*?)\n?\s*```/m))
    begin
      JSON.parse(match[1])
    rescue JSON::ParserError
      nil
    end
  end
end