Module: RubyLLM::Contract

Defined in:
lib/ruby_llm/contract.rb,
lib/ruby_llm/contract/dsl.rb,
lib/ruby_llm/contract/types.rb,
lib/ruby_llm/contract/errors.rb,
lib/ruby_llm/contract/railtie.rb,
lib/ruby_llm/contract/version.rb,
lib/ruby_llm/contract/minitest.rb,
lib/ruby_llm/contract/step/dsl.rb,
lib/ruby_llm/contract/rake_task.rb,
lib/ruby_llm/contract/step/base.rb,
lib/ruby_llm/contract/prompt/ast.rb,
lib/ruby_llm/contract/step/trace.rb,
lib/ruby_llm/contract/eval/report.rb,
lib/ruby_llm/contract/eval/runner.rb,
lib/ruby_llm/contract/prompt/node.rb,
lib/ruby_llm/contract/step/result.rb,
lib/ruby_llm/contract/step/runner.rb,
lib/ruby_llm/contract/eval/dataset.rb,
lib/ruby_llm/contract/adapters/base.rb,
lib/ruby_llm/contract/adapters/test.rb,
lib/ruby_llm/contract/configuration.rb,
lib/ruby_llm/contract/pipeline/base.rb,
lib/ruby_llm/contract/rspec/helpers.rb,
lib/ruby_llm/contract/pipeline/trace.rb,
lib/ruby_llm/contract/prompt/builder.rb,
lib/ruby_llm/contract/contract/parser.rb,
lib/ruby_llm/contract/cost_calculator.rb,
lib/ruby_llm/contract/pipeline/result.rb,
lib/ruby_llm/contract/pipeline/runner.rb,
lib/ruby_llm/contract/prompt/renderer.rb,
lib/ruby_llm/contract/rspec/pass_eval.rb,
lib/ruby_llm/contract/token_estimator.rb,
lib/ruby_llm/contract/eval/case_result.rb,
lib/ruby_llm/contract/eval/case_scorer.rb,
lib/ruby_llm/contract/eval/prompt_diff.rb,
lib/ruby_llm/contract/eval/recommender.rb,
lib/ruby_llm/contract/adapters/response.rb,
lib/ruby_llm/contract/adapters/ruby_llm.rb,
lib/ruby_llm/contract/eval/eval_history.rb,
lib/ruby_llm/contract/eval/report_stats.rb,
lib/ruby_llm/contract/step/retry_policy.rb,
lib/ruby_llm/contract/concerns/eval_host.rb,
lib/ruby_llm/contract/contract/invariant.rb,
lib/ruby_llm/contract/contract/validator.rb,
lib/ruby_llm/contract/eval/baseline_diff.rb,
lib/ruby_llm/contract/eval/case_executor.rb,
lib/ruby_llm/contract/step/limit_checker.rb,
lib/ruby_llm/contract/step/runner_config.rb,
lib/ruby_llm/contract/contract/definition.rb,
lib/ruby_llm/contract/eval/recommendation.rb,
lib/ruby_llm/contract/eval/report_storage.rb,
lib/ruby_llm/contract/step/adapter_caller.rb,
lib/ruby_llm/contract/step/result_builder.rb,
lib/ruby_llm/contract/step/retry_executor.rb,
lib/ruby_llm/contract/concerns/deep_freeze.rb,
lib/ruby_llm/contract/eval/eval_definition.rb,
lib/ruby_llm/contract/eval/evaluator/exact.rb,
lib/ruby_llm/contract/eval/evaluator/regex.rb,
lib/ruby_llm/contract/eval/retry_optimizer.rb,
lib/ruby_llm/contract/eval/trait_evaluator.rb,
lib/ruby_llm/contract/step/input_validator.rb,
lib/ruby_llm/contract/step/prompt_compiler.rb,
lib/ruby_llm/contract/eval/model_comparison.rb,
lib/ruby_llm/contract/eval/report_presenter.rb,
lib/ruby_llm/contract/eval/evaluation_result.rb,
lib/ruby_llm/contract/prompt/nodes/rule_node.rb,
lib/ruby_llm/contract/prompt/nodes/user_node.rb,
lib/ruby_llm/contract/concerns/deep_symbolize.rb,
lib/ruby_llm/contract/concerns/trace_equality.rb,
lib/ruby_llm/contract/concerns/context_helpers.rb,
lib/ruby_llm/contract/eval/case_result_builder.rb,
lib/ruby_llm/contract/prompt/nodes/system_node.rb,
lib/ruby_llm/contract/concerns/usage_aggregator.rb,
lib/ruby_llm/contract/contract/schema_validator.rb,
lib/ruby_llm/contract/prompt/nodes/example_node.rb,
lib/ruby_llm/contract/prompt/nodes/section_node.rb,
lib/ruby_llm/contract/eval/expectation_evaluator.rb,
lib/ruby_llm/contract/eval/prompt_diff_presenter.rb,
lib/ruby_llm/contract/eval/prompt_diff_comparator.rb,
lib/ruby_llm/contract/eval/prompt_diff_serializer.rb,
lib/ruby_llm/contract/eval/step_result_normalizer.rb,
lib/ruby_llm/contract/eval/contract_detail_builder.rb,
lib/ruby_llm/contract/eval/evaluator/json_includes.rb,
lib/ruby_llm/contract/eval/pipeline_result_adapter.rb,
lib/ruby_llm/contract/eval/evaluator/proc_evaluator.rb,
lib/ruby_llm/contract/eval/step_expectation_applier.rb,
lib/ruby_llm/contract/contract/schema_validator/node.rb,
lib/ruby_llm/contract/contract/schema_validator/enum_rule.rb,
lib/ruby_llm/contract/contract/schema_validator/type_rule.rb,
lib/ruby_llm/contract/contract/schema_validator/bound_rule.rb,
lib/ruby_llm/contract/contract/schema_validator/object_rules.rb,
lib/ruby_llm/contract/contract/schema_validator/scalar_rules.rb,
lib/ruby_llm/contract/contract/schema_validator/schema_extractor.rb

Defined Under Namespace

Modules: Adapters, Concerns, CostCalculator, DSL, Eval, MinitestHelpers, Pipeline, Prompt, RSpec, Step, StepAdapterOverride, TokenEstimator, Types Classes: AdapterError, Configuration, ContractError, Definition, Error, InputError, Invariant, OptimizeRakeTask, ParseError, Parser, Railtie, RakeTask, SchemaValidator, Validator

Constant Summary collapse

VERSION = 
"0.6.2"

Class Method Summary collapse

Class Method Details

.configuration ⇒ Object 



10
11
12
 
# File 'lib/ruby_llm/contract.rb', line 10

def configuration
  @configuration ||= Configuration.new
end

.configure {|configuration| ... } ⇒ Object

Yields:



14
15
16
17
 
# File 'lib/ruby_llm/contract.rb', line 14

def configure
  yield(configuration)
  auto_create_adapter! if configuration.default_adapter.nil?
end

.eval_hosts ⇒ Object 



30
31
32
 
# File 'lib/ruby_llm/contract.rb', line 30

def eval_hosts
  @eval_hosts ||= []
end

.load_evals!(*dirs) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/ruby_llm/contract.rb', line 53

def load_evals!(*dirs)
  dirs = dirs.flatten.compact
  if dirs.empty? && defined?(::Rails)
    dirs = %w[app/steps/eval app/contracts/eval].filter_map do |path|
      full = ::Rails.root.join(path)
      full.to_s if full.exist?
    end
  end

  return if dirs.empty?

  # In Rails, eager-load parent directories so contract classes
  # are available when eval files reference them.
  eager_load_contract_dirs! if defined?(::Rails)

  # Clear file-sourced evals ONCE, then load ALL dirs.
  Thread.current[:ruby_llm_contract_reloading] = true
  eval_hosts.each do |host|
    host.clear_file_sourced_evals! if host.respond_to?(:clear_file_sourced_evals!)
  end

  dirs.each do |d|
    Dir[File.join(d, "**", "*_eval.rb")].each { |f| load f }
  end
ensure
  Thread.current[:ruby_llm_contract_reloading] = false
end

.normalize_candidate_config(entry) ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/ruby_llm/contract.rb', line 81

def normalize_candidate_config(entry)
  case entry
  when String
    raise ArgumentError, "Candidate model must be a non-empty String" if entry.strip.empty?

    { model: entry.strip }
  when Hash
    model = entry[:model] || entry["model"]
    unless model.is_a?(String) && !model.strip.empty?
      raise ArgumentError, "Candidate config must include a non-empty String :model"
    end

    normalized = { model: model.strip }
    effort = entry[:reasoning_effort] || entry["reasoning_effort"]
    normalized[:reasoning_effort] = effort if effort
    normalized
  else
    raise ArgumentError, "Expected String or Hash, got #{entry.class}"
  end
end

.register_eval_host(klass) ⇒ Object

— Eval host registry —



26
27
28
 
# File 'lib/ruby_llm/contract.rb', line 26

def register_eval_host(klass)
  eval_hosts << klass unless eval_hosts.include?(klass)
end

.reset_configuration! ⇒ Object 



19
20
21
22
 
# File 'lib/ruby_llm/contract.rb', line 19

def reset_configuration!
  @configuration = Configuration.new
  step_adapter_overrides.clear
end

.reset_eval_hosts! ⇒ Object 



40
41
42
 
# File 'lib/ruby_llm/contract.rb', line 40

def reset_eval_hosts!
  @eval_hosts = []
end

.run_all_evals(context: {}) ⇒ Object 



34
35
36
37
38
 
# File 'lib/ruby_llm/contract.rb', line 34

def run_all_evals(context: {})
  live_eval_hosts.to_h do |host|
    [host, host.run_eval(context: context)]
  end
end

.step_adapter_overrides ⇒ Object

Thread-local per-step adapter overrides used by test helpers (RSpec + Minitest).



45
46
47
 
# File 'lib/ruby_llm/contract.rb', line 45

def step_adapter_overrides
  Thread.current[:ruby_llm_contract_step_overrides] ||= {}
end

.step_adapter_overrides=(map) ⇒ Object 



49
50
51
 
# File 'lib/ruby_llm/contract.rb', line 49

def step_adapter_overrides=(map)
  Thread.current[:ruby_llm_contract_step_overrides] = map
end