Class: RubricLLM::Judge

Inherits:
Object
  • Object
show all
Defined in:
lib/rubric_llm/judge.rb

Constant Summary collapse

METRIC_RESPONSE_SCHEMA = 
{
  name: "rubric_llm_metric_response",
  strict: false,
  schema: {
    type: "object",
    properties: {
      score: { type: "number", minimum: 0.0, maximum: 1.0 },
      reasoning: { type: "string" },
      claims: { type: "array", items: { type: "object" } },
      context_scores: { type: "array", items: { type: "object" } },
      covered_facts: { type: "array", items: { type: "object" } },
      discrepancies: { type: "array", items: { type: "object" } }
    },
    required: ["score"],
    additionalProperties: true
  }
}.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config:) ⇒ Judge

Returns a new instance of Judge.



27
28
29
 
# File 'lib/rubric_llm/judge.rb', line 27

def initialize(config:)
  @config = config
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



25
26
27
 
# File 'lib/rubric_llm/judge.rb', line 25

def config
  @config
end

Instance Method Details

#call(system_prompt:, user_prompt:) ⇒ Object

Send a prompt to the judge LLM and return the parsed JSON response. Retries transient failures with exponential backoff.



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/rubric_llm/judge.rb', line 33

def call(system_prompt:, user_prompt:)
  config.validate!
  attempts = 0
  begin
    attempts += 1
    chat = RubyLLM.chat(model: config.judge_model, provider: config.judge_provider)
    chat.with_temperature(config.temperature)
    chat.with_params(max_tokens: config.max_tokens)
    apply_response_schema(chat)

    full_system_prompt = build_system_prompt(system_prompt)
    chat.with_instructions(full_system_prompt)
    response = chat.ask(user_prompt)
    content = response.content
    validate_response!(content.is_a?(Hash) ? content : parse_json(content))
  rescue StandardError => e
    if attempts > config.max_retries
      raise e if e.is_a?(JudgeError)

      raise JudgeError, "Judge call failed: #{e.message}"
    end

    sleep(config.retry_base_delay * (2**(attempts - 1)))
    retry
  end
end

#parse_json(text) ⇒ Object

Parse JSON from LLM output with multiple strategies:

  1. Direct JSON.parse
  2. Extract from markdown code fence
  3. Raise JudgeError for malformed output


64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/rubric_llm/judge.rb', line 64

def parse_json(text)
  raise JudgeError, "Judge response was empty" if text.nil? || text.strip.empty?

  JSON.parse(text)
rescue JSON::ParserError => e
  if (match = text.match(/```(?:json)?\s*\n?(.*?)\n?\s*```/m))
    begin
      return JSON.parse(match[1])
    rescue JSON::ParserError => e
      raise JudgeError, "Judge response code fence was not valid JSON: #{e.message}"
    end
  end

  raise JudgeError, "Judge response was not valid JSON: #{e.message}"
end