Class: RubyLLM::Contract::Eval::RetryOptimizer

Inherits:
Object
  • Object
show all
Defined in:
lib/ruby_llm/contract/eval/retry_optimizer.rb

Overview

Runs compare_models on ALL evals for a step, builds a score matrix, identifies the constraining eval, and suggests an escalation chain.

optimizer = RetryOptimizer.new(step: MyStep, candidates: [...], context: {})
result = optimizer.call
result.print_summary
result.to_dsl  # => copy-paste retry_policy

Defined Under Namespace

Classes: Result

Instance Method Summary collapse

Constructor Details

#initialize(step:, candidates:, context: {}, min_score: 0.95) ⇒ RetryOptimizer

Returns a new instance of RetryOptimizer.



97
98
99
100
101
102
 
# File 'lib/ruby_llm/contract/eval/retry_optimizer.rb', line 97

def initialize(step:, candidates:, context: {}, min_score: 0.95)
  @step = step
  @candidates = candidates
  @context = context
  @min_score = min_score
end

Instance Method Details

#call ⇒ Object 



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/ruby_llm/contract/eval/retry_optimizer.rb', line 104

def call
  evals = @step.eval_names
  return empty_result(evals) if evals.empty?

  score_matrix = {}
  evals.each do |eval_name|
    comparison = with_retry_disabled do
      @step.compare_models(eval_name, candidates: @candidates, context: @context)
    end
    score_matrix[eval_name] = extract_scores(comparison)
  end

  labels = score_matrix.values.flat_map(&:keys).uniq
  constraining = find_constraining_eval(score_matrix, labels)
  chain, details = build_chain(score_matrix, labels, evals)

  Result.new(
    step_name: @step.name || @step.to_s,
    eval_names: evals,
    candidate_labels: labels,
    score_matrix: score_matrix,
    constraining_eval: constraining,
    chain: chain,
    chain_details: details
  )
end