Class: AgentEvalPlanner::SuiteValidator

Inherits:
Object
  • Object
show all
Defined in:
lib/agent_eval_planner/suite_validator.rb

Overview

Hard-fail validator for suite JSONL (Ruby port of the skill script).

Constant Summary collapse

PLACEHOLDER_RE = 
/\{\{[^{}]+\}\}/
GUARDRAIL_INTENTS = 
%w[
  out_of_scope_refusal
  prompt_injection
  role_escape
  prompt_exfiltration
  pii_probe
].freeze
GUARDRAIL_TAGS = 
%w[scope injection role exfil pii guardrail].freeze

Instance Method Summary collapse

Constructor Details

#initialize(path:, known_tools: [], agent_has_no_tools: false) ⇒ SuiteValidator

Returns a new instance of SuiteValidator.



19
20
21
22
23
 
# File 'lib/agent_eval_planner/suite_validator.rb', line 19

def initialize(path:, known_tools: [], agent_has_no_tools: false)
  @path = path
  @known_tools = Array(known_tools).map(&:to_s).reject(&:empty?)
  @agent_has_no_tools = agent_has_no_tools
end

Instance Method Details

#validate ⇒ Object

Raises:



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/agent_eval_planner/suite_validator.rb', line 32

def validate
  raise InputError, "Arquivo não encontrado: #{path}" unless File.file?(path)

  raw = File.read(path, encoding: "UTF-8")
  errors = placeholder_errors(raw) + mode_errors
  known = known_tools.to_set
  seen_ids = {}

  raw.each_line.with_index(1) do |line, line_no|
    stripped = line.strip
    next if stripped.empty?

    row, load_errors = load_row(stripped, line_no)
    if row.nil?
      errors.concat(load_errors)
      next
    end

    row_id, id_errors = row_id_errors(row, line_no, seen_ids)
    errors.concat(id_errors)
    next if row_id.nil?

    errors.concat(errors_for_row(row, row_id, known: known))
  end

  errors
end

#validate! ⇒ Object

Raises:



25
26
27
28
29
30
 
# File 'lib/agent_eval_planner/suite_validator.rb', line 25

def validate!
  errors = validate
  raise ValidationError, errors.join("\n") if errors.any?

  true
end