Class: RubricLLM::Evaluator

Inherits:
Object
  • Object
show all
Defined in:
lib/rubric_llm/evaluator.rb

Constant Summary collapse

DEFAULT_METRICS = 
[
  Metrics::Faithfulness,
  Metrics::Relevance,
  Metrics::Correctness,
  Metrics::FactualAccuracy,
  Metrics::ContextPrecision,
  Metrics::ContextRecall
].freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config:, metrics: nil) ⇒ Evaluator

Returns a new instance of Evaluator.



16
17
18
19
 
# File 'lib/rubric_llm/evaluator.rb', line 16

def initialize(config:, metrics: nil)
  @config = config
  @metric_classes = metrics || DEFAULT_METRICS
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



14
15
16
 
# File 'lib/rubric_llm/evaluator.rb', line 14

def config
  @config
end

#metric_classes ⇒ Object (readonly)

Returns the value of attribute metric_classes.



14
15
16
 
# File 'lib/rubric_llm/evaluator.rb', line 14

def metric_classes
  @metric_classes
end

Instance Method Details

#call(question:, answer:, context: [], ground_truth: nil) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/rubric_llm/evaluator.rb', line 21

def call(question:, answer:, context: [], ground_truth: nil)
  judge = Judge.new(config:)
  scores = {}
  details = {}

  metric_classes.each do |metric_class|
    metric = metric_class.new(judge:)
    name = metric_name(metric_class)

    result = metric.call(question:, answer:, context:, ground_truth:)
    scores[name] = result[:score]
    details[name] = result[:details]
  rescue StandardError => e
    scores[name] = nil
    details[name] = { error: e.message }
  end

  Result.new(scores:, details:, sample: { question:, answer:, context:, ground_truth: })
end