Class: RubyLLM::Contract::Eval::EvalDefinition

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/contract/eval/eval_definition.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, step_class: nil) ⇒ EvalDefinition

Returns a new instance of EvalDefinition.



9
10
11
12
13
14
15
16
 
# File 'lib/ruby_llm/contract/eval/eval_definition.rb', line 9

def initialize(name, step_class: nil, &)
  @name = name
  @step_class = step_class
  @default_input = nil
  @sample_response = nil
  @cases = []
  instance_eval(&)
end

Instance Attribute Details

#cases ⇒ Object (readonly)

Returns the value of attribute cases.



7
8
9
 
# File 'lib/ruby_llm/contract/eval/eval_definition.rb', line 7

def cases
  @cases
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



7
8
9
 
# File 'lib/ruby_llm/contract/eval/eval_definition.rb', line 7

def name
  @name
end

Instance Method Details

#add_case(description, input: nil, expected: nil, expected_traits: nil, evaluator: nil, step_expectations: nil) ⇒ Object

Raises:

  • (ArgumentError) 


34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/ruby_llm/contract/eval/eval_definition.rb', line 34

def add_case(description, input: nil, expected: nil, expected_traits: nil, evaluator: nil, step_expectations: nil)
  case_input = input.nil? ? @default_input : input
  raise ArgumentError, "add_case requires input (set default_input or pass input:)" if case_input.nil?
  validate_unique_case_name!(description)

  @cases << {
    name: description,
    input: case_input,
    expected: expected,
    expected_traits: expected_traits,
    evaluator: evaluator,
    step_expectations: step_expectations
  }
end

#build_adapter ⇒ Object 



28
29
30
31
32
 
# File 'lib/ruby_llm/contract/eval/eval_definition.rb', line 28

def build_adapter
  return nil unless defined?(@has_sample_response) && @has_sample_response

  Adapters::Test.new(response: @sample_response)
end

#build_dataset ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/ruby_llm/contract/eval/eval_definition.rb', line 69

def build_dataset
  eval_cases = effective_cases
  eval_name = @name
  Dataset.define(eval_name) do
    eval_cases.each do |eval_case|
      add_case(eval_case[:name], input: eval_case[:input], expected: eval_case[:expected],
                                 expected_traits: eval_case[:expected_traits],
                                 evaluator: eval_case[:evaluator],
                                 step_expectations: eval_case[:step_expectations])
    end
  end
end

#default_input(input) ⇒ Object 



18
19
20
 
# File 'lib/ruby_llm/contract/eval/eval_definition.rb', line 18

def default_input(input)
  @default_input = input
end

#sample_response(response) ⇒ Object 



22
23
24
25
26
 
# File 'lib/ruby_llm/contract/eval/eval_definition.rb', line 22

def sample_response(response)
  @sample_response = response
  @has_sample_response = true
  pre_validate_sample! if @step_class
end

#verify(description, expected_or_proc = nil, input: nil, expect: nil) ⇒ Object 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/ruby_llm/contract/eval/eval_definition.rb', line 49

def verify(description, expected_or_proc = nil, input: nil, expect: nil)
  if !expected_or_proc.nil? && !expect.nil?
    raise ArgumentError, "verify accepts either a positional argument or expect: keyword, not both"
  end

  expected_or_proc = expect unless expect.nil?
  case_input = input.nil? ? @default_input : input
  validate_verify_args!(expected_or_proc, case_input)
  validate_unique_case_name!(description)

  evaluator = expected_or_proc.is_a?(::Proc) ? expected_or_proc : nil

  @cases << {
    name: description,
    input: case_input,
    expected: evaluator ? nil : expected_or_proc,
    evaluator: evaluator
  }
end