Module: RubyLLM::Contract::Concerns::EvalHost

Includes:
ContextHelpers
Included in:
Pipeline::Base, Step::Base
Defined in:
lib/ruby_llm/contract/concerns/eval_host.rb

Instance Method Summary collapse

Instance Method Details

#clear_file_sourced_evals! ⇒ Object 



24
25
26
27
28
29
 
# File 'lib/ruby_llm/contract/concerns/eval_host.rb', line 24

def clear_file_sourced_evals!
  return unless defined?(@file_sourced_evals) && defined?(@eval_definitions)

  @file_sourced_evals.each { |key| @eval_definitions.delete(key) }
  @file_sourced_evals.clear
end

#compare_models(eval_name, models:, context: {}) ⇒ Object 



48
49
50
51
52
53
54
55
56
 
# File 'lib/ruby_llm/contract/concerns/eval_host.rb', line 48

def compare_models(eval_name, models:, context: {})
  context = safe_context(context)
  models = models.uniq
  reports = models.each_with_object({}) do |model, hash|
    model_context = isolate_context(context).merge(model: model)
    hash[model] = run_single_eval(eval_name, model_context)
  end
  Eval::ModelComparison.new(eval_name: eval_name, reports: reports)
end

#define_eval(name) ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/ruby_llm/contract/concerns/eval_host.rb', line 8

def define_eval(name, &)
  @eval_definitions ||= {}
  @file_sourced_evals ||= Set.new
  key = name.to_s

  if @eval_definitions.key?(key) && !Thread.current[:ruby_llm_contract_reloading]
    warn "[ruby_llm-contract] Redefining eval '#{key}' on #{self}. " \
         "This replaces the previous definition."
  end

  @eval_definitions[key] = Eval::EvalDefinition.new(key, step_class: self, &)
  @file_sourced_evals.add(key) if Thread.current[:ruby_llm_contract_reloading]
  Contract.register_eval_host(self)
  register_subclasses(self)
end

#eval_defined? ⇒ Boolean

Returns:

  • (Boolean) 


35
36
37
 
# File 'lib/ruby_llm/contract/concerns/eval_host.rb', line 35

def eval_defined?
  !all_eval_definitions.empty?
end

#eval_names ⇒ Object 



31
32
33
 
# File 'lib/ruby_llm/contract/concerns/eval_host.rb', line 31

def eval_names
  all_eval_definitions.keys
end

#run_eval(name = nil, context: {}, concurrency: nil) ⇒ Object 



39
40
41
42
43
44
45
46
 
# File 'lib/ruby_llm/contract/concerns/eval_host.rb', line 39

def run_eval(name = nil, context: {}, concurrency: nil)
  context = safe_context(context)
  if name
    run_single_eval(name, context, concurrency: concurrency)
  else
    run_all_own_evals(context, concurrency: concurrency)
  end
end