Class: RubyLLM::Contract::Step::Base

Inherits:
Object
  • Object
show all
Extended by:
Concerns::ContextHelpers, Concerns::EvalHost, Dsl, RetryExecutor
Defined in:
lib/ruby_llm/contract/step/base.rb

Constant Summary collapse

DEFAULT_OUTPUT_TOKENS = 
256
KNOWN_CONTEXT_KEYS = 
%i[adapter model temperature max_tokens provider assume_model_exists
reasoning_effort retry_policy_override attachment].freeze

Constants included from Dsl

Dsl::UNSET

Constants included from Concerns::EvalHost

Concerns::EvalHost::SAMPLE_RESPONSE_COMPARE_WARNING

Class Method Summary collapse

Methods included from Dsl

around_call, attachment_token_estimate, class_observers, class_validates, contract, inherited_value, inherited_value_with_reset, input_type, max_cost, max_input, max_output, model, observe, on_unknown_attachment_size, on_unknown_pricing, output_schema, output_type, prompt, reasoning_effort, retry_policy, temperature, thinking, validate

Methods included from Concerns::EvalHost

clear_file_sourced_evals!, compare_models, compare_with, define_eval, eval_defined?, eval_names, run_eval

Class Method Details

.build_messages(input) ⇒ Object 



102
103
104
105
106
107
 
# File 'lib/ruby_llm/contract/step/base.rb', line 102

def build_messages(input)
  dynamic = prompt.arity >= 1
  builder_input = dynamic ? input : Prompt::Builder::NOT_PROVIDED
  ast = Prompt::Builder.build(input: builder_input, &prompt)
  Prompt::Renderer.render(ast, variables: prompt_variables(input, dynamic))
end

.estimate_cost(input:, model: nil, attachment: nil) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/ruby_llm/contract/step/base.rb', line 24

def estimate_cost(input:, model: nil, attachment: nil)
  model_name = estimated_model_name(model)
  model_info = CostCalculator.find_model(model_name)
  return nil unless model_info

  text_tokens = TokenEstimator.estimate(build_messages(input))
  attachment_tokens, attachment_error = resolve_attachment_tokens(attachment)
  return nil if attachment_error

  input_tokens = text_tokens + attachment_tokens
  # NOTE: attachment tokens add to input only, not output. Vision-
  # heavy outputs (long image descriptions) may exceed
  # `output_tokens_estimate` — this method is a floor for budget
  # planning, not a precise predictor. See multimodal_input.md.
  output_tokens = max_output || DEFAULT_OUTPUT_TOKENS

  {
    model: model_name,
    input_tokens: input_tokens,
    output_tokens_estimate: output_tokens,
    estimated_cost: CostCalculator.calculate(
      model_name: model_name,
      usage: { input_tokens: input_tokens, output_tokens: output_tokens }
    )
  }
end

.estimate_eval_cost(eval_name, models: nil) ⇒ Object

Raises:

  • (ArgumentError) 


51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/ruby_llm/contract/step/base.rb', line 51

def estimate_eval_cost(eval_name, models: nil)
  defn = send(:all_eval_definitions)[eval_name.to_s]
  raise ArgumentError, "No eval '#{eval_name}' defined" unless defn

  model_list = models || [estimated_model_name].compact
  cases = defn.build_dataset.cases

  model_list.each_with_object({}) do |model_name, result|
    result[model_name] = estimate_eval_cost_for_model(cases, model_name)
  end
end

.eval_case(input:, expected: nil, expected_traits: nil, evaluator: nil, context: {}) ⇒ Object 



19
20
21
22
 
# File 'lib/ruby_llm/contract/step/base.rb', line 19

def eval_case(input:, expected: nil, expected_traits: nil, evaluator: nil, context: {})
  Eval::Runner.run(step: self, dataset: inline_dataset(input, expected, expected_traits, evaluator),
                   context: context).results.first
end

.inherited(subclass) ⇒ Object 



9
10
11
12
 
# File 'lib/ruby_llm/contract/step/base.rb', line 9

def self.inherited(subclass)
  super
  Contract.register_eval_host(subclass) if respond_to?(:eval_defined?) && eval_defined?
end

.optimize_retry_policy(candidates:, context: {}, min_score: Eval::DEFAULT_MIN_SCORE, runs: 1, production_mode: nil) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/ruby_llm/contract/step/base.rb', line 75

def optimize_retry_policy(candidates:, context: {},
                          min_score: Eval::DEFAULT_MIN_SCORE,
                          runs: 1, production_mode: nil)
  Eval::RetryOptimizer.new(
    step: self,
    candidates: candidates,
    context: context,
    min_score: min_score,
    runs: runs,
    production_mode: production_mode
  ).call
end

.recommend(eval_name, candidates:, context: {}, min_score: Eval::DEFAULT_MIN_SCORE, min_first_try_pass_rate: Eval::DEFAULT_MIN_FIRST_TRY_PASS_RATE) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/ruby_llm/contract/step/base.rb', line 63

def recommend(eval_name, candidates:, context: {},
              min_score: Eval::DEFAULT_MIN_SCORE,
              min_first_try_pass_rate: Eval::DEFAULT_MIN_FIRST_TRY_PASS_RATE)
  comparison = compare_models(eval_name, candidates: candidates, context: context)
  Eval::Recommender.new(
    comparison: comparison,
    min_score: min_score,
    min_first_try_pass_rate: min_first_try_pass_rate,
    current_config: current_model_config
  ).recommend
end

.run(input, context: {}) ⇒ Object 



93
94
95
96
97
98
99
100
 
# File 'lib/ruby_llm/contract/step/base.rb', line 93

def run(input, context: {})
  context = safe_context(context)
  warn_unknown_context_keys(context)

  result = dispatch_run(input, context)
  log_result(result)
  invoke_around_call(input, result)
end