Class: Qualspec::Suite::Definition

Inherits:
Object
  • Object
show all
Defined in:
lib/qualspec/suite/dsl.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, &block) ⇒ Definition

Returns a new instance of Definition.



8
9
10
11
12
13
14
15
16
17
 
# File 'lib/qualspec/suite/dsl.rb', line 8

def initialize(name, &block)
  @name = name
  @candidates_list = []
  @scenarios_list = []
  @variants_config = nil
  @temperature_list = [nil] # nil means use model default
  @track_cost = false

  instance_eval(&block) if block_given? # rubocop:disable Style/EvalWithLocation -- DSL pattern requires eval
end

Instance Attribute Details

#candidates_list ⇒ Object (readonly)

Returns the value of attribute candidates_list.



6
7
8
 
# File 'lib/qualspec/suite/dsl.rb', line 6

def candidates_list
  @candidates_list
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



6
7
8
 
# File 'lib/qualspec/suite/dsl.rb', line 6

def name
  @name
end

#scenarios_list ⇒ Object (readonly)

Returns the value of attribute scenarios_list.



6
7
8
 
# File 'lib/qualspec/suite/dsl.rb', line 6

def scenarios_list
  @scenarios_list
end

#temperature_list ⇒ Object (readonly)

Returns the value of attribute temperature_list.



6
7
8
 
# File 'lib/qualspec/suite/dsl.rb', line 6

def temperature_list
  @temperature_list
end

#variants_config ⇒ Object (readonly)

Returns the value of attribute variants_config.



6
7
8
 
# File 'lib/qualspec/suite/dsl.rb', line 6

def variants_config
  @variants_config
end

Instance Method Details

#behaves_like(behavior_name) ⇒ Object Also known as: it_behaves_like, include_behavior

DSL: include shared behaviors



78
79
80
81
 
# File 'lib/qualspec/suite/dsl.rb', line 78

def behaves_like(behavior_name)
  behavior = Behavior.find(behavior_name)
  @scenarios_list.concat(behavior.scenarios_list)
end

#candidate(name, model: nil, system_prompt: nil, **options) ⇒ Object 



38
39
40
 
# File 'lib/qualspec/suite/dsl.rb', line 38

def candidate(name, model: nil, system_prompt: nil, **options)
  @candidates_list << Candidate.new(name, model: model, system_prompt: system_prompt, **options)
end

#candidates(&block) ⇒ Object

DSL: define candidates



34
35
36
 
# File 'lib/qualspec/suite/dsl.rb', line 34

def candidates(&block)
  instance_eval(&block) # rubocop:disable Style/EvalWithLocation -- DSL pattern requires eval
end

#scenario(name, &block) ⇒ Object

DSL: define scenarios



43
44
45
 
# File 'lib/qualspec/suite/dsl.rb', line 43

def scenario(name, &block)
  @scenarios_list << Scenario.new(name, &block)
end

#temperatures(temps) ⇒ Object

DSL: define temperatures to test across

Examples:

temperatures [0.0, 0.7, 1.0]


67
68
69
70
71
72
73
74
75
 
# File 'lib/qualspec/suite/dsl.rb', line 67

def temperatures(temps)
  temps = Array(temps)
  temps.each do |t|
    next if t.nil?
    raise ArgumentError, "Temperature must be numeric, got #{t.inspect}" unless t.is_a?(Numeric)
    raise ArgumentError, "Temperature #{t} outside valid range 0.0-2.0" unless (0.0..2.0).cover?(t)
  end
  @temperature_list = temps
end

#track_cost(value = true) ⇒ Object Also known as: capture_metadata

DSL: capture per-call cost + token metadata so cost/value analysis works. Off by default — evaluations that don't look at cost skip the overhead.

Examples:

track_cost


24
25
26
 
# File 'lib/qualspec/suite/dsl.rb', line 24

def track_cost(value = true) # rubocop:disable Style/OptionalBooleanParameter -- reads as a DSL toggle
  @track_cost = value
end

#track_cost? ⇒ Boolean

Returns:

  • (Boolean) 


29
30
31
 
# File 'lib/qualspec/suite/dsl.rb', line 29

def track_cost?
  @track_cost
end

#variants(factory: :prompt_variant, &block) ⇒ Object

DSL: configure variants using FactoryBot

Examples:

Using trait matrix for combinatorial testing

variants factory: :prompt_variant do
  trait_matrix [:msw, :layperson], [:neutral, :concerned]
end

Using explicit variant definitions

variants do
  variant :expert_concerned, traits: [:msw, :concerned]
  variant :naive_neutral, traits: [:layperson, :neutral]
end


59
60
61
 
# File 'lib/qualspec/suite/dsl.rb', line 59

def variants(factory: :prompt_variant, &block)
  @variants_config = VariantsConfig.new(factory: factory, &block)
end