Module: Ask::Eval

Defined in:
lib/ask/eval.rb,
lib/ask/eval/dsl.rb,
lib/ask/eval/judge.rb,
lib/ask/eval/runner.rb,
lib/ask/eval/version.rb,
lib/ask/eval/recorder.rb,
lib/ask/eval/test_case.rb,
lib/ask/eval/assertions.rb,
lib/ask/eval/judges/bias.rb,
lib/ask/eval/cost_tracker.rb,
lib/ask/eval/session_eval.rb,
lib/ask/eval/configuration.rb,
lib/ask/eval/judges/faithful.rb,
lib/ask/eval/judges/toxicity.rb,
lib/ask/eval/reporters/junit.rb,
lib/ask/eval/assertions/judge.rb,
lib/ask/eval/reporters/github.rb,
lib/ask/eval/reporters/console.rb,
lib/ask/eval/judges/correctness.rb,
lib/ask/eval/judges/hallucination.rb,
lib/ask/eval/assertions/deterministic.rb

Defined Under Namespace

Modules: Assertions, DSL, Judges, Reporters Classes: AssertionError, Configuration, CostTracker, Error, Judge, Recorder, Runner, SessionEval, TestCase

Constant Summary collapse

VERSION = 
"0.2.0"

Class Method Summary collapse

Class Method Details

.configuration ⇒ Configuration

Returns the global configuration.

Returns:



57
58
59
 
# File 'lib/ask/eval.rb', line 57

def configuration
  @configuration ||= Configuration.new
end

.configure {|Configuration| ... } ⇒ Object

Configure ask-eval globally.

Yields:



52
53
54
 
# File 'lib/ask/eval.rb', line 52

def configure
  yield configuration
end

.cost_report ⇒ Hash

Returns cost report from all evaluations.

Returns:

  • (Hash) —

    cost report from all evaluations



62
63
64
 
# File 'lib/ask/eval.rb', line 62

def cost_report
  configuration.cost_report
end

.evaluate(output, assertions, context: nil, input: nil, expected: nil) ⇒ Array<Hash>

Run a batch evaluation.

Parameters:

  • output (String) —

    the LLM output

  • assertions (Array<Hash>) —

    array of assertion configs

  • context (String, Array<String>, nil) (defaults to: nil) —

    source context

  • input (String, nil) (defaults to: nil) —

    input/prompt

  • expected (String, nil) (defaults to: nil) —

    expected output

Returns:

  • (Array<Hash>) —

    array of results



40
41
42
43
44
45
46
47
48
 
# File 'lib/ask/eval.rb', line 40

def evaluate(output, assertions, context: nil, input: nil, expected: nil)
  test_case = TestCase.new(
    actual_output: output,
    input: input,
    context: context,
    expected_output: expected
  )
  Assertions.evaluate_all(test_case, assertions)
end