Module: RubyLLM::Contract

Defined in:
lib/ruby_llm/contract.rb,
lib/ruby_llm/contract/dsl.rb,
lib/ruby_llm/contract/types.rb,
lib/ruby_llm/contract/errors.rb,
lib/ruby_llm/contract/railtie.rb,
lib/ruby_llm/contract/version.rb,
lib/ruby_llm/contract/minitest.rb,
lib/ruby_llm/contract/step/dsl.rb,
lib/ruby_llm/contract/rake_task.rb,
lib/ruby_llm/contract/step/base.rb,
lib/ruby_llm/contract/prompt/ast.rb,
lib/ruby_llm/contract/step/trace.rb,
lib/ruby_llm/contract/eval/report.rb,
lib/ruby_llm/contract/eval/runner.rb,
lib/ruby_llm/contract/prompt/node.rb,
lib/ruby_llm/contract/step/result.rb,
lib/ruby_llm/contract/step/runner.rb,
lib/ruby_llm/contract/eval/dataset.rb,
lib/ruby_llm/contract/adapters/base.rb,
lib/ruby_llm/contract/adapters/test.rb,
lib/ruby_llm/contract/configuration.rb,
lib/ruby_llm/contract/pipeline/base.rb,
lib/ruby_llm/contract/rspec/helpers.rb,
lib/ruby_llm/contract/pipeline/trace.rb,
lib/ruby_llm/contract/prompt/builder.rb,
lib/ruby_llm/contract/contract/parser.rb,
lib/ruby_llm/contract/cost_calculator.rb,
lib/ruby_llm/contract/pipeline/result.rb,
lib/ruby_llm/contract/pipeline/runner.rb,
lib/ruby_llm/contract/prompt/renderer.rb,
lib/ruby_llm/contract/rspec/pass_eval.rb,
lib/ruby_llm/contract/token_estimator.rb,
lib/ruby_llm/contract/eval/case_result.rb,
lib/ruby_llm/contract/adapters/response.rb,
lib/ruby_llm/contract/adapters/ruby_llm.rb,
lib/ruby_llm/contract/eval/eval_history.rb,
lib/ruby_llm/contract/step/retry_policy.rb,
lib/ruby_llm/contract/concerns/eval_host.rb,
lib/ruby_llm/contract/contract/invariant.rb,
lib/ruby_llm/contract/contract/validator.rb,
lib/ruby_llm/contract/eval/baseline_diff.rb,
lib/ruby_llm/contract/step/limit_checker.rb,
lib/ruby_llm/contract/contract/definition.rb,
lib/ruby_llm/contract/step/retry_executor.rb,
lib/ruby_llm/contract/concerns/deep_freeze.rb,
lib/ruby_llm/contract/eval/eval_definition.rb,
lib/ruby_llm/contract/eval/evaluator/exact.rb,
lib/ruby_llm/contract/eval/evaluator/regex.rb,
lib/ruby_llm/contract/eval/trait_evaluator.rb,
lib/ruby_llm/contract/eval/model_comparison.rb,
lib/ruby_llm/contract/eval/evaluation_result.rb,
lib/ruby_llm/contract/prompt/nodes/rule_node.rb,
lib/ruby_llm/contract/prompt/nodes/user_node.rb,
lib/ruby_llm/contract/concerns/deep_symbolize.rb,
lib/ruby_llm/contract/concerns/trace_equality.rb,
lib/ruby_llm/contract/concerns/context_helpers.rb,
lib/ruby_llm/contract/prompt/nodes/system_node.rb,
lib/ruby_llm/contract/concerns/usage_aggregator.rb,
lib/ruby_llm/contract/contract/schema_validator.rb,
lib/ruby_llm/contract/prompt/nodes/example_node.rb,
lib/ruby_llm/contract/prompt/nodes/section_node.rb,
lib/ruby_llm/contract/eval/contract_detail_builder.rb,
lib/ruby_llm/contract/eval/evaluator/json_includes.rb,
lib/ruby_llm/contract/eval/pipeline_result_adapter.rb,
lib/ruby_llm/contract/eval/evaluator/proc_evaluator.rb

Defined Under Namespace

Modules: Adapters, Concerns, CostCalculator, DSL, Eval, MinitestHelpers, Pipeline, Prompt, RSpec, Step, TokenEstimator, Types Classes: AdapterError, Configuration, ContractError, Definition, Error, InputError, Invariant, ParseError, Parser, Railtie, RakeTask, SchemaValidator, Validator

Constant Summary collapse

VERSION = 
"0.4.0"

Class Method Summary collapse

Class Method Details

.configuration ⇒ Object 



10
11
12
 
# File 'lib/ruby_llm/contract.rb', line 10

def configuration
  @configuration ||= Configuration.new
end

.configure {|configuration| ... } ⇒ Object

Yields:



14
15
16
17
 
# File 'lib/ruby_llm/contract.rb', line 14

def configure
  yield(configuration)
  auto_create_adapter! if configuration.default_adapter.nil?
end

.eval_hosts ⇒ Object 



29
30
31
 
# File 'lib/ruby_llm/contract.rb', line 29

def eval_hosts
  @eval_hosts ||= []
end

.load_evals!(*dirs) ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/ruby_llm/contract.rb', line 43

def load_evals!(*dirs)
  dirs = dirs.flatten.compact
  if dirs.empty? && defined?(::Rails)
    dirs = %w[app/steps/eval app/contracts/eval].filter_map do |path|
      full = ::Rails.root.join(path)
      full.to_s if full.exist?
    end
  end

  return if dirs.empty?

  # In Rails, eager-load parent directories so contract classes
  # are available when eval files reference them.
  eager_load_contract_dirs! if defined?(::Rails)

  # Clear file-sourced evals ONCE, then load ALL dirs.
  Thread.current[:ruby_llm_contract_reloading] = true
  eval_hosts.each do |host|
    host.clear_file_sourced_evals! if host.respond_to?(:clear_file_sourced_evals!)
  end

  dirs.each do |d|
    Dir[File.join(d, "**", "*_eval.rb")].each { |f| load f }
  end
ensure
  Thread.current[:ruby_llm_contract_reloading] = false
end

.register_eval_host(klass) ⇒ Object

— Eval host registry —



25
26
27
 
# File 'lib/ruby_llm/contract.rb', line 25

def register_eval_host(klass)
  eval_hosts << klass unless eval_hosts.include?(klass)
end

.reset_configuration! ⇒ Object 



19
20
21
 
# File 'lib/ruby_llm/contract.rb', line 19

def reset_configuration!
  @configuration = Configuration.new
end

.reset_eval_hosts! ⇒ Object 



39
40
41
 
# File 'lib/ruby_llm/contract.rb', line 39

def reset_eval_hosts!
  @eval_hosts = []
end

.run_all_evals(context: {}) ⇒ Object 



33
34
35
36
37
 
# File 'lib/ruby_llm/contract.rb', line 33

def run_all_evals(context: {})
  live_eval_hosts.to_h do |host|
    [host, host.run_eval(context: context)]
  end
end