Class: Riffer::Evals::Judge

Inherits:
Object
  • Object
show all
Defined in:
lib/riffer/evals/judge.rb,
sig/generated/riffer/evals/judge.rbs

Overview

Executes LLM-as-judge evaluations, using tool calling internally to get structured output from the judge model.

Defined Under Namespace

Classes: EvaluationTool

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(model:, provider_options: {}) ⇒ Judge

Raises Riffer::ArgumentError unless model is "provider/model" format.

: (model: String, ?provider_options: Hash[Symbol, untyped]) -> void

Parameters:

  • model: (String)
  • provider_options: (Hash[Symbol, untyped]) (defaults to: {}) 


37
38
39
40
41
42
43
44
45
 
# File 'lib/riffer/evals/judge.rb', line 37

def initialize(model:, provider_options: {})
  provider_name, model_name = model.split("/", 2)
  unless [provider_name, model_name].all? { |part| part.is_a?(String) && !part.strip.empty? }
    raise Riffer::ArgumentError, "Invalid model string: #{model}"
  end

  @model = model
  @provider_options = provider_options
end

Instance Attribute Details

#model ⇒ String (readonly)

The model string (provider/model format).

Returns:

  • (String) 


32
33
34
 
# File 'lib/riffer/evals/judge.rb', line 32

def model
  @model
end

Instance Method Details

#build_system_message(instructions) ⇒ String

-- : (String) -> String

Parameters:

  • (String)

Returns:

  • (String) 


68
69
70
71
72
73
74
75
76
 
# File 'lib/riffer/evals/judge.rb', line 68

def build_system_message(instructions)
  <<~SYSTEM.strip
    You are an evaluation assistant. Score the output based on the instructions below.

    #{instructions}

    Use the evaluation tool to submit your score and reasoning.
  SYSTEM
end

#build_user_message(input:, output:, ground_truth: nil) ⇒ String

-- : (input: String, output: String, ?ground_truth: String?) -> String

Parameters:

  • input: (String)
  • output: (String)
  • ground_truth: (String, nil) (defaults to: nil)

Returns:

  • (String) 


80
81
82
83
84
85
86
 
# File 'lib/riffer/evals/judge.rb', line 80

def build_user_message(input:, output:, ground_truth: nil)
  parts = [] #: Array[String]
  parts << "## Input\n\n#{input}"
  parts << "## Output\n\n#{output}"
  parts << "## Ground Truth\n\n#{ground_truth}" if ground_truth
  parts.join("\n\n")
end

#evaluate(instructions:, input:, output:, ground_truth: nil) ⇒ Hash[Symbol, untyped]

Evaluates an input/output pair using the configured LLM.

: (instructions: String, input: String, output: String, ?ground_truth: String?) -> Hash[Symbol, untyped]

Parameters:

  • instructions: (String)
  • input: (String)
  • output: (String)
  • ground_truth: (String, nil) (defaults to: nil)

Returns:

  • (Hash[Symbol, untyped]) 


50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/riffer/evals/judge.rb', line 50

def evaluate(instructions:, input:, output:, ground_truth: nil)
  system_message = build_system_message(instructions)
  user_message = build_user_message(input: input, output: output, ground_truth: ground_truth)

  response = provider_instance.generate_text(
    system: system_message,
    prompt: user_message,
    model: model_name,
    tools: [EvaluationTool]
  )

  parse_tool_response(response)
end

#model_name ⇒ String

-- : () -> String

Returns:

  • (String) 


106
107
108
 
# File 'lib/riffer/evals/judge.rb', line 106

def model_name
  @model_name ||= @model.split("/", 2).last
end

#parse_tool_response(response) ⇒ Hash[Symbol, untyped]

-- : (Riffer::Messages::Assistant) -> Hash[Symbol, untyped]

Parameters:

Returns:

  • (Hash[Symbol, untyped]) 


112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/riffer/evals/judge.rb', line 112

def parse_tool_response(response)
  tool_call = response.tool_calls.first
  raise Riffer::Error, "Invalid judge response: no tool call found" unless tool_call

  parsed = JSON.parse(tool_call[:arguments], symbolize_names: true)
  score = parsed[:score]
  reason = parsed[:reason]

  raise Riffer::Error, "Invalid judge response: missing score" if score.nil?

  {
    score: score.to_f,
    reason: reason
  }
rescue JSON::ParserError => e
  raise Riffer::Error, "Invalid judge response: #{e.message}"
end

#provider_instance ⇒ Riffer::Providers::Base

-- : () -> Riffer::Providers::Base

Returns:



90
91
92
93
94
95
96
 
# File 'lib/riffer/evals/judge.rb', line 90

def provider_instance
  @provider_instance ||= begin
    provider_class = Riffer::Providers::Repository.find(provider_name)
    raise Riffer::ArgumentError, "Provider not found: #{provider_name}" unless provider_class
    provider_class.new(**@provider_options)
  end
end

#provider_name ⇒ String

-- : () -> String

Returns:

  • (String) 


100
101
102
 
# File 'lib/riffer/evals/judge.rb', line 100

def provider_name
  @provider_name ||= @model.split("/", 2).first
end