Module: RobotLab::To::Evals

Defined in:
lib/robot_lab/to/evals/base.rb,
lib/robot_lab/to/evals/code.rb,
lib/robot_lab/to/evals/null.rb,
lib/robot_lab/to/evals/prose.rb,
lib/robot_lab/to/evals/score.rb,
lib/robot_lab/to/evals/context.rb,
lib/robot_lab/to/evals/factory.rb

Defined Under Namespace

Classes: Base, Code, Context, Null, ProcEval, Prose, Score

Class Method Summary collapse

Class Method Details

.blank?(value) ⇒ Boolean

Returns:

  • (Boolean) 


75
 
# File 'lib/robot_lab/to/evals/factory.rb', line 75

def self.blank?(value) = presence(value).nil?

.build(config, git: CommitManager.new) ⇒ Object

Resolves the configured eval strategy into an object responding to #score(context). Precedence: an explicit instance (responds to #score) or proc, a registered symbol, then the named built-ins, then a default that matches legacy behavior (Code when a verify command is set, else Null).

Phase 1 ships "code" and "null"; "prose", file paths, and "auto" arrive in later phases (they raise a clear error until then).



25
26
27
28
29
30
31
 
# File 'lib/robot_lab/to/evals/factory.rb', line 25

def self.build(config, git: CommitManager.new)
  chosen = config.respond_to?(:eval) ? config.eval : nil
  return chosen if chosen.respond_to?(:score)
  return ProcEval.new(chosen) if chosen.is_a?(Proc)

  from_name(chosen, config, git)
end

.code(config) ⇒ Object 



56
57
58
59
60
61
 
# File 'lib/robot_lab/to/evals/factory.rb', line 56

def self.code(config)
  Code.new(verify:  presence(config.verify_command),
           measure: presence(config.eval_measure),
           target:  config.eval_target,
           timeout: config.verify_timeout)
end

.from_name(chosen, config, git) ⇒ Object 



33
34
35
36
37
38
39
40
41
42
 
# File 'lib/robot_lab/to/evals/factory.rb', line 33

def self.from_name(chosen, config, git)
  case chosen
  when Symbol then from_registry(chosen, config)
  when "code" then code(config)
  when "null" then Null.new
  when "prose" then prose(config, git)
  when nil    then use_code?(config) ? code(config) : Null.new
  else raise ArgumentError, unknown_strategy(chosen)
  end
end

.from_registry(name, config) ⇒ Object

Raises:

  • (ArgumentError) 


63
64
65
66
67
68
 
# File 'lib/robot_lab/to/evals/factory.rb', line 63

def self.from_registry(name, config)
  block = RobotLab::To.evals[name]
  raise ArgumentError, "no eval registered as #{name.inspect}" unless block

  block.call(config)
end

.presence(value) ⇒ Object 



70
71
72
73
 
# File 'lib/robot_lab/to/evals/factory.rb', line 70

def self.presence(value)
  text = value.to_s.strip
  text.empty? ? nil : text
end

.prose(config, git) ⇒ Object 



44
45
46
47
48
 
# File 'lib/robot_lab/to/evals/factory.rb', line 44

def self.prose(config, git)
  Prose.new(objective_model: config.model, git: git,
            spec: config.eval_spec, floor: config.eval_floor,
            judge_model: config.eval_judge_model || config.model)
end

.unknown_strategy(chosen) ⇒ Object 



77
78
79
80
81
 
# File 'lib/robot_lab/to/evals/factory.rb', line 77

def self.unknown_strategy(chosen)
  "unknown eval strategy: #{chosen.inspect} (available: code, null, " \
    "a symbol registered via RobotLab::To.register_eval, " \
    "an object responding to #score, or a proc)"
end

.use_code?(config) ⇒ Boolean

Default to Code when any of its inputs are configured (a verify command, a measure command, or a target); otherwise Null preserves legacy behavior.

Returns:

  • (Boolean) 


52
53
54
 
# File 'lib/robot_lab/to/evals/factory.rb', line 52

def self.use_code?(config)
  !blank?(config.verify_command) || !config.eval_measure.nil? || !config.eval_target.nil?
end