Class: RubyLLM::Contract::Step::Base

Inherits:
Object
  • Object
show all
Extended by:
Concerns::ContextHelpers, Concerns::EvalHost, Dsl, RetryExecutor
Defined in:
lib/ruby_llm/contract/step/base.rb

Constant Summary collapse

DEFAULT_OUTPUT_TOKENS = 
256
KNOWN_CONTEXT_KEYS = 
%i[adapter model temperature max_tokens provider assume_model_exists reasoning_effort].freeze

Constants included from Concerns::EvalHost

Concerns::EvalHost::SAMPLE_RESPONSE_COMPARE_WARNING

Class Method Summary collapse

Methods included from Dsl

around_call, class_observers, class_validates, contract, input_type, max_cost, max_input, max_output, model, observe, on_unknown_pricing, output_schema, output_type, prompt, retry_policy, temperature, validate

Methods included from Concerns::EvalHost

clear_file_sourced_evals!, compare_models, compare_with, define_eval, eval_defined?, eval_names, run_eval

Class Method Details

.build_messages(input) ⇒ Object 



84
85
86
87
88
89
 
# File 'lib/ruby_llm/contract/step/base.rb', line 84

def build_messages(input)
  dynamic = prompt.arity >= 1
  builder_input = dynamic ? input : Prompt::Builder::NOT_PROVIDED
  ast = Prompt::Builder.build(input: builder_input, &prompt)
  Prompt::Renderer.render(ast, variables: prompt_variables(input, dynamic))
end

.estimate_cost(input:, model: nil) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/ruby_llm/contract/step/base.rb', line 24

def estimate_cost(input:, model: nil)
  model_name = estimated_model_name(model)
  model_info = CostCalculator.send(:find_model, model_name)
  return nil unless model_info

  input_tokens = TokenEstimator.estimate(build_messages(input))
  output_tokens = max_output || DEFAULT_OUTPUT_TOKENS

  {
    model: model_name,
    input_tokens: input_tokens,
    output_tokens_estimate: output_tokens,
    estimated_cost: estimated_cost_for(model_info, input_tokens, output_tokens)
  }
end

.estimate_eval_cost(eval_name, models: nil) ⇒ Object

Raises:

  • (ArgumentError) 


40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/ruby_llm/contract/step/base.rb', line 40

def estimate_eval_cost(eval_name, models: nil)
  defn = send(:all_eval_definitions)[eval_name.to_s]
  raise ArgumentError, "No eval '#{eval_name}' defined" unless defn

  model_list = models || [estimated_model_name].compact
  cases = defn.build_dataset.cases

  model_list.each_with_object({}) do |model_name, result|
    result[model_name] = estimate_eval_cost_for_model(cases, model_name)
  end
end

.eval_case(input:, expected: nil, expected_traits: nil, evaluator: nil, context: {}) ⇒ Object 



19
20
21
22
 
# File 'lib/ruby_llm/contract/step/base.rb', line 19

def eval_case(input:, expected: nil, expected_traits: nil, evaluator: nil, context: {})
  Eval::Runner.run(step: self, dataset: inline_dataset(input, expected, expected_traits, evaluator),
                   context: context).results.first
end

.inherited(subclass) ⇒ Object 



9
10
11
12
 
# File 'lib/ruby_llm/contract/step/base.rb', line 9

def self.inherited(subclass)
  super
  Contract.register_eval_host(subclass) if respond_to?(:eval_defined?) && eval_defined?
end

.optimize_retry_policy(candidates:, context: {}, min_score: 0.95) ⇒ Object 



62
63
64
65
66
67
68
69
 
# File 'lib/ruby_llm/contract/step/base.rb', line 62

def optimize_retry_policy(candidates:, context: {}, min_score: 0.95)
  Eval::RetryOptimizer.new(
    step: self,
    candidates: candidates,
    context: context,
    min_score: min_score
  ).call
end

.recommend(eval_name, candidates:, min_score: 0.95, min_first_try_pass_rate: 0.8, context: {}) ⇒ Object 



52
53
54
55
56
57
58
59
60
 
# File 'lib/ruby_llm/contract/step/base.rb', line 52

def recommend(eval_name, candidates:, min_score: 0.95, min_first_try_pass_rate: 0.8, context: {})
  comparison = compare_models(eval_name, candidates: candidates, context: context)
  Eval::Recommender.new(
    comparison: comparison,
    min_score: min_score,
    min_first_try_pass_rate: min_first_try_pass_rate,
    current_config: current_model_config
  ).recommend
end

.run(input, context: {}) ⇒ Object 



75
76
77
78
79
80
81
82
 
# File 'lib/ruby_llm/contract/step/base.rb', line 75

def run(input, context: {})
  context = safe_context(context)
  warn_unknown_context_keys(context)

  result = dispatch_run(input, context)
  log_result(result)
  invoke_around_call(input, result)
end