Module: Phronomy::Eval

Defined in:

lib/phronomy/eval.rb,
lib/phronomy/eval/runner.rb,
lib/phronomy/eval/scorer.rb,
lib/phronomy/eval/dataset.rb,
lib/phronomy/eval/metrics.rb,
lib/phronomy/eval/eval_case.rb,
lib/phronomy/eval/comparison.rb,
lib/phronomy/eval/eval_result.rb,
lib/phronomy/eval/scorer/base.rb,
lib/phronomy/eval/scorer/llm_judge.rb,
lib/phronomy/eval/scorer/exact_match.rb,
lib/phronomy/eval/scorer/includes_scorer.rb

Overview

Namespace module for the evaluation framework.

Defined Under Namespace

Modules: Scorer Classes: Comparison, Dataset, EvalCase, EvalResult, Metrics, Runner

Instance Attribute Summary

• #actual ⇒ Object readonly

  the callable's output.

• #latency_ms ⇒ Object readonly

  wall-clock time of the callable in ms.

• #usage ⇒ Object readonly

  An immutable record holding the outcome of evaluating one EvalCase.

Instance Method Summary

• #eval_case(value) ⇒ Object

  the original sample.

• #score(value) ⇒ Object

  scorer-assigned value in [0.0, 1.0].

Instance Attribute Details

#actual ⇒ Object (readonly)

the callable's output

# File 'lib/phronomy/eval/eval_result.rb', line 12

EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms) do
  # Returns true when the scorer assigned a perfect score of 1.0.
  def pass?
    score >= 1.0
  end
end

#latency_ms ⇒ Object (readonly)

wall-clock time of the callable in ms

# File 'lib/phronomy/eval/eval_result.rb', line 12

EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms) do
  # Returns true when the scorer assigned a perfect score of 1.0.
  def pass?
    score >= 1.0
  end
end

#usage ⇒ Object (readonly)

An immutable record holding the outcome of evaluating one EvalCase.

# File 'lib/phronomy/eval/eval_result.rb', line 12

EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms) do
  # Returns true when the scorer assigned a perfect score of 1.0.
  def pass?
    score >= 1.0
  end
end

Instance Method Details

#eval_case=(value) ⇒ Object

the original sample

# File 'lib/phronomy/eval/eval_result.rb', line 12

EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms) do
  # Returns true when the scorer assigned a perfect score of 1.0.
  def pass?
    score >= 1.0
  end
end

#score=(value) ⇒ Object

scorer-assigned value in [0.0, 1.0]

# File 'lib/phronomy/eval/eval_result.rb', line 12

EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms) do
  # Returns true when the scorer assigned a perfect score of 1.0.
  def pass?
    score >= 1.0
  end
end