Module: Ask::Eval::Assertions

Defined in:
lib/ask/eval/assertions.rb,
lib/ask/eval/assertions/judge.rb,
lib/ask/eval/assertions/deterministic.rb

Defined Under Namespace

Modules: Deterministic, Judge

Class Method Summary collapse

Class Method Details

.available_assertions ⇒ Array<Symbol>

Returns all available assertion names.

Returns:

  • (Array<Symbol>) —

    all available assertion names



40
41
42
 
# File 'lib/ask/eval/assertions.rb', line 40

def available_assertions
  DETERMINISTIC_ASSERTIONS + JUDGE_ASSERTIONS
end

.deterministic_assertions ⇒ Array<Symbol>

Returns deterministic assertion names.

Returns:

  • (Array<Symbol>) —

    deterministic assertion names



45
46
47
 
# File 'lib/ask/eval/assertions.rb', line 45

def deterministic_assertions
  DETERMINISTIC_ASSERTIONS
end

.evaluate(name, output, **kwargs) ⇒ Ask::Eval::Judge::Result, Hash

Evaluate a single assertion by name.

Parameters:

  • name (Symbol) —

    assertion name (:contains, :faithful, etc.)

  • output (String) —

    the LLM output

  • kwargs (Hash) —

    additional arguments

Returns:



13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/ask/eval/assertions.rb', line 13

def evaluate(name, output, **kwargs)
  case name
  when *DETERMINISTIC_ASSERTIONS
    evaluate_deterministic(name, output, **kwargs)
  when *JUDGE_ASSERTIONS
    evaluate_judge(name, output, **kwargs)
  else
    raise ArgumentError, "Unknown assertion: #{name.inspect}. " \
                         "Available: #{available_assertions.join(', ')}"
  end
end

.evaluate_all(test_case, assertions) ⇒ Array<Hash>

Evaluate multiple assertions against a test case.

Parameters:

Returns:

  • (Array<Hash>) —

    results with assertion name and result



30
31
32
33
34
35
36
37
 
# File 'lib/ask/eval/assertions.rb', line 30

def evaluate_all(test_case, assertions)
  assertions.map do |assertion|
    name = assertion[:name] || assertion["name"]
    kwargs = assertion.reject { |k, _| k.to_s == "name" }
    result = evaluate(name, test_case.actual_output, **kwargs)
    { name: name, result: result }
  end
end

.judge_assertions ⇒ Array<Symbol>

Returns judge-based assertion names.

Returns:

  • (Array<Symbol>) —

    judge-based assertion names



50
51
52
 
# File 'lib/ask/eval/assertions.rb', line 50

def judge_assertions
  JUDGE_ASSERTIONS
end