Class: RubyLLM::Contract::Step::Base

Inherits:
Object
  • Object
show all
Extended by:
Concerns::ContextHelpers, Concerns::EvalHost, Dsl, RetryExecutor
Defined in:
lib/ruby_llm/contract/step/base.rb

Constant Summary collapse

KNOWN_CONTEXT_KEYS = 
%i[adapter model temperature max_tokens provider assume_model_exists].freeze

Class Method Summary collapse

Methods included from Dsl

around_call, class_validates, contract, input_type, max_cost, max_input, max_output, model, output_schema, output_type, prompt, retry_policy, temperature, validate

Methods included from Concerns::EvalHost

clear_file_sourced_evals!, compare_models, define_eval, eval_defined?, eval_names, run_eval

Class Method Details

.build_messages(input) ⇒ Object 



86
87
88
89
90
91
92
93
 
# File 'lib/ruby_llm/contract/step/base.rb', line 86

def build_messages(input)
  dynamic = prompt.arity >= 1
  builder_input = dynamic ? input : Prompt::Builder::NOT_PROVIDED
  ast = Prompt::Builder.build(input: builder_input, &prompt)
  variables = dynamic ? {} : { input: input }
  variables.merge!(input.transform_keys(&:to_sym)) if !dynamic && input.is_a?(Hash)
  Prompt::Renderer.render(ast, variables: variables)
end

.estimate_cost(input:, model: nil) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/ruby_llm/contract/step/base.rb', line 26

def estimate_cost(input:, model: nil)
  model_name = model || RubyLLM::Contract.configuration.default_model
  messages = build_messages(input)
  input_tokens = TokenEstimator.estimate(messages)
  output_tokens = max_output || 256 # conservative default

  model_info = CostCalculator.send(:find_model, model_name)
  return nil unless model_info

  estimated = CostCalculator.send(:compute_cost, model_info,
                                  { input_tokens: input_tokens, output_tokens: output_tokens })
  {
    model: model_name,
    input_tokens: input_tokens,
    output_tokens_estimate: output_tokens,
    estimated_cost: estimated
  }
end

.estimate_eval_cost(eval_name, models: nil) ⇒ Object

Raises:

  • (ArgumentError) 


45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/ruby_llm/contract/step/base.rb', line 45

def estimate_eval_cost(eval_name, models: nil)
  defn = send(:all_eval_definitions)[eval_name.to_s]
  raise ArgumentError, "No eval '#{eval_name}' defined" unless defn

  model_list = models || [RubyLLM::Contract.configuration.default_model].compact
  cases = defn.build_dataset.cases

  model_list.each_with_object({}) do |model_name, result|
    per_case = cases.sum do |c|
      est = estimate_cost(input: c.input, model: model_name)
      est ? est[:estimated_cost] : 0.0
    end
    result[model_name] = per_case.round(6)
  end
end

.eval_case(input:, expected: nil, expected_traits: nil, evaluator: nil, context: {}) ⇒ Object 



17
18
19
20
21
22
23
24
 
# File 'lib/ruby_llm/contract/step/base.rb', line 17

def eval_case(input:, expected: nil, expected_traits: nil, evaluator: nil, context: {})
  dataset = Eval::Dataset.define("single_case") do
    add_case("inline", input: input, expected: expected,
                       expected_traits: expected_traits, evaluator: evaluator)
  end
  report = Eval::Runner.run(step: self, dataset: dataset, context: context)
  report.results.first
end

.inherited(subclass) ⇒ Object 



7
8
9
10
 
# File 'lib/ruby_llm/contract/step/base.rb', line 7

def self.inherited(subclass)
  super
  Contract.register_eval_host(subclass) if respond_to?(:eval_defined?) && eval_defined?
end

.run(input, context: {}) ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/ruby_llm/contract/step/base.rb', line 65

def run(input, context: {})
  context = safe_context(context)
  warn_unknown_context_keys(context)
  adapter = resolve_adapter(context)
  default_model = context[:model] || model || RubyLLM::Contract.configuration.default_model
  policy = retry_policy

  ctx_temp = context[:temperature]
  extra = context.slice(:provider, :assume_model_exists, :max_tokens)
  result = if policy
             run_with_retry(input, adapter: adapter, default_model: default_model,
                            policy: policy, context_temperature: ctx_temp, extra_options: extra)
           else
             run_once(input, adapter: adapter, model: default_model,
                      context_temperature: ctx_temp, extra_options: extra)
           end

  log_result(result)
  invoke_around_call(input, result)
end