Class: ActiveHarness::Tribunal

Inherits:
Object
  • Object
show all
Includes:
Core::HookRunner
Defined in:
lib/active_harness/tribunal.rb,
lib/active_harness/tribunal/dsl.rb,
lib/active_harness/tribunal/hooks.rb,
lib/active_harness/tribunal/processing.rb

Overview

Can be used directly or subclassed with a class-level DSL.

Direct usage:

tribunal = ActiveHarness::Tribunal.new(
  input:   "Is this message toxic?",
  context: { user_id: 42 },
  agents:  [ToxicityAgent, BiasAgent, SpamAgent],
  timeout: 7
)
tribunal.on(:after_agent) { |result| puts result.model }
tribunal.process { |results| results.all? { |r| r.processed["result"] == true } }
tribunal.call

Subclass with DSL:

class ContentQualityTribunal < ActiveHarness::Tribunal
  agents PolitenessAgent, ConstructivenessAgent
  on(:after_agent) { |result| puts result.model }
  process { |results| results.all? { |r| r.processed["result"] == true } }
end
ContentQualityTribunal.new(input: "...").call

Direct Known Subclasses

SupportGuardTribunal

Constant Summary collapse

VALID_STRATEGIES =

Declarative verdict — built-in aggregation strategy with a per-result evaluator.

Strategies:

:unanimous  — verdict true when every successful result evaluates to true
:majority   — verdict true when more than half of successful results evaluate to true

Options:

may_fail: N — tolerate up to N agent errors before raising AllAgentsFailed
              (default: nil — raise only when all agents fail, preserving legacy behavior)

The block receives a single Result and must return a truthy/falsy value.

verdict :unanimous do |result|
  result.processed["result"] == true
end

verdict :majority, may_fail: 1 do |result|
  result.processed["result"] == true
end

%i[unanimous majority].freeze
VALID_HOOKS = 
%i[
  before_call
  before_agent
  after_agent
  agent_error
  after_call
  before_verdict
  after_verdict
].freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input: nil, context: {}, params: {}, memory: nil, agents: nil, timeout: 7, streams: {}, may_fail: :_unset) ⇒ Tribunal

Returns a new instance of Tribunal.



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/active_harness/tribunal.rb', line 62

def initialize(
  input:    nil,
  context:  {},
  params:   {},
  memory:   nil,
  agents:   nil,
  timeout:  7,
  streams:  {},
  may_fail: :_unset
)
  config = self.class.tribunal_config

  @input                 = input
  @context               = context
  @params                = params
  @memory                = memory
  @agents                = agents || config[:agents]
  @timeout               = timeout
  @process_block         = config[:process]
  @strategy              = config[:strategy]
  @evaluate_block        = config[:evaluate_block]
  @may_fail              = may_fail == :_unset ? config[:may_fail] : may_fail
  @hooks                 = config[:hooks].transform_values { |v| Array(v).dup }
  @token_stream          = streams[:token]
  @agent_event_stream    = streams[:agent]
  @tribunal_event_stream = streams[:tribunal]
  @results               = []
  @errors                = []
  @verdict               = nil
  @execution_time        = nil
  @agent_execution_times = []
end

Instance Attribute Details

#agent_event_stream ⇒ Object (readonly)

Returns the value of attribute agent_event_stream.



53
54
55
 
# File 'lib/active_harness/tribunal.rb', line 53

def agent_event_stream
  @agent_event_stream
end

#agent_execution_times ⇒ Object (readonly)

Returns the value of attribute agent_execution_times.



53
54
55
 
# File 'lib/active_harness/tribunal.rb', line 53

def agent_execution_times
  @agent_execution_times
end

#context ⇒ Object

Instance API



49
50
51
 
# File 'lib/active_harness/tribunal.rb', line 49

def context
  @context
end

#errors ⇒ Object (readonly)

Returns the value of attribute errors.



53
54
55
 
# File 'lib/active_harness/tribunal.rb', line 53

def errors
  @errors
end

#execution_time ⇒ Object (readonly)

Returns the value of attribute execution_time.



53
54
55
 
# File 'lib/active_harness/tribunal.rb', line 53

def execution_time
  @execution_time
end

#input ⇒ Object

Instance API



49
50
51
 
# File 'lib/active_harness/tribunal.rb', line 49

def input
  @input
end

#memory ⇒ Object

Instance API



49
50
51
 
# File 'lib/active_harness/tribunal.rb', line 49

def memory
  @memory
end

#params ⇒ Object

Instance API



49
50
51
 
# File 'lib/active_harness/tribunal.rb', line 49

def params
  @params
end

#results ⇒ Object (readonly)

Returns the value of attribute results.



53
54
55
 
# File 'lib/active_harness/tribunal.rb', line 53

def results
  @results
end

#token_stream ⇒ Object (readonly)

Returns the value of attribute token_stream.



53
54
55
 
# File 'lib/active_harness/tribunal.rb', line 53

def token_stream
  @token_stream
end

#tribunal_event_stream ⇒ Object (readonly)

Returns the value of attribute tribunal_event_stream.



53
54
55
 
# File 'lib/active_harness/tribunal.rb', line 53

def tribunal_event_stream
  @tribunal_event_stream
end

#verdict ⇒ Object (readonly)

Returns the value of attribute verdict.



53
54
55
 
# File 'lib/active_harness/tribunal.rb', line 53

def verdict
  @verdict
end

Class Method Details

.after(event, &block) ⇒ Object 



45
46
47
 
# File 'lib/active_harness/tribunal/hooks.rb', line 45

def after(event, &block)
  on(:"after_#{event}", &block)
end

.agents(*list) ⇒ Object

Declare agents at the class level.

agents PolitenessAgent, ConstructivenessAgent
agents [PolitenessAgent, ConstructivenessAgent]



8
9
10
 
# File 'lib/active_harness/tribunal/dsl.rb', line 8

def agents(*list)
  tribunal_config[:agents] = list.flatten
end

.before(event, &block) ⇒ Object

Rails-style aliases for on:

before :call                     do ... end       # → on :before_call
before :agent                    do |agent| end   # → on :before_agent
before :verdict                  do |results| end # → on :before_verdict (transform)
after  :call                     do |r, e| end    # → on :after_call
after  :agent                    do |result| end  # → on :after_agent
after  :verdict                  do |verdict| end # → on :after_verdict
callback :agent_error            do |name, e| end # → on :agent_error



41
42
43
 
# File 'lib/active_harness/tribunal/hooks.rb', line 41

def before(event, &block)
  on(:"before_#{event}", &block)
end

.callback(event, &block) ⇒ Object 



49
50
51
 
# File 'lib/active_harness/tribunal/hooks.rb', line 49

def callback(event, &block)
  on(event, &block)
end

.inherited(subclass) ⇒ Object 



41
42
43
 
# File 'lib/active_harness/tribunal.rb', line 41

def inherited(subclass)
  subclass.instance_variable_set(:@tribunal_config, { agents: [], hooks: {} })
end

.on(event, &block) ⇒ Object

Class-level hook registration.

on :before_call                  do ... end
on :before_agent                 do |agent| ... end
on :after_agent                  do |result| ... end
on :agent_error                  do |name, error| ... end
on :after_call                   do |results, errors| ... end
on :before_verdict               do |results| results end  # transform hook
on :after_verdict                do |verdict| ... end



23
24
25
26
27
28
29
30
 
# File 'lib/active_harness/tribunal/hooks.rb', line 23

def on(event, &block)
  unless VALID_HOOKS.include?(event)
    raise ArgumentError,
      "Unknown Tribunal hook :#{event}. Valid hooks: #{VALID_HOOKS.map { |h| ":#{h}" }.join(", ")}"
  end

  (tribunal_config[:hooks][event] ||= []) << block
end

.process(&block) ⇒ Object

Class-level process block — defines how to compute the verdict from all results. Receives the full results array; return value becomes #verdict. Takes priority over verdict strategy if both are declared.

process { |results| results.all? { |r| r.processed["result"] == true } }



17
18
19
 
# File 'lib/active_harness/tribunal/dsl.rb', line 17

def process(&block)
  tribunal_config[:process] = block
end

.tribunal_config ⇒ Object

Each subclass gets its own isolated config hash.



37
38
39
 
# File 'lib/active_harness/tribunal.rb', line 37

def tribunal_config
  @tribunal_config ||= { agents: [], hooks: {} }
end

.verdict(strategy, may_fail: nil, &block) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
 
# File 'lib/active_harness/tribunal/dsl.rb', line 42

def verdict(strategy, may_fail: nil, &block)
  unless VALID_STRATEGIES.include?(strategy)
    raise ArgumentError,
      "Unknown verdict strategy :#{strategy}. Valid strategies: #{VALID_STRATEGIES.map { |s| ":#{s}" }.join(", ")}"
  end

  tribunal_config[:strategy]       = strategy
  tribunal_config[:may_fail]       = may_fail
  tribunal_config[:evaluate_block] = block
end

Instance Method Details

#call ⇒ Object

Run all agents in parallel, then compute the verdict. Returns self so calls can be chained: tribunal.call.verdict

Behaviour on failure:

- If some agents fail/timeout, their errors are in #errors and
  #results contains only successful results.
- If ALL agents fail/timeout, raises Errors::AllAgentsFailed.



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/active_harness/tribunal.rb', line 113

def call
  agents = resolve_agents
  fire(:before_call)

  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)

  futures = agents.each_with_index.map do |agent, index|
    fire(:before_agent, agent, index)
    t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
    future = Concurrent::Future.execute { agent.call }
    [future, t0]
  end

  @results               = []
  @errors                = []
  @agent_execution_times = []

  futures.each_with_index do |(future, t0), index|
    future.wait(@timeout)
    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0).round(3)
    @agent_execution_times << { agent: agents[index].class.name, time: elapsed }

    if future.fulfilled?
      value  = future.value
      result = value.is_a?(ActiveHarness::Agent) ? value.result : value
      @results << result
      fire(:after_agent, result, index)
    elsif future.incomplete?
      error = Errors::TimeoutError.new(
        "Agent #{agents[index].class.name} timed out after #{@timeout}s"
      )
      @errors << { agent: agents[index].class.name, error: error }
      fire(:agent_error, agents[index].class.name, error, index)
    else
      @errors << { agent: agents[index].class.name, error: future.reason }
      fire(:agent_error, agents[index].class.name, future.reason, index)
    end
  end

  @execution_time = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at).round(3)

  fire(:after_call, @results, @errors)

  # If all agents failed, raise an exception.
  # Otherwise, compute the verdict based on successful results.
  check_failure_threshold!

  verdict_input = transform_hook(:before_verdict, @results)
  @verdict      = compute_verdict(verdict_input)

  fire(:after_verdict, @verdict)

  self
end

#on(event, &block) ⇒ Object

Instance-level hook registration — appends to class-level hooks for this event. :before_verdict is a transform hook: its return value replaces the results array passed to the process block.



57
58
59
60
61
62
63
64
65
 
# File 'lib/active_harness/tribunal/hooks.rb', line 57

def on(event, &block)
  unless VALID_HOOKS.include?(event)
    raise ArgumentError,
      "Unknown Tribunal hook :#{event}. Valid hooks: #{VALID_HOOKS.map { |h| ":#{h}" }.join(", ")}"
  end

  (@hooks[event] ||= []) << block
  self
end

#process(&block) ⇒ Object

Instance-level process block — overrides class-level block.

tribunal.process { |results| results.count { |r| r.processed["ok"] } >= 2 }



6
7
8
9
 
# File 'lib/active_harness/tribunal/processing.rb', line 6

def process(&block)
  @process_block = block
  self
end

#result ⇒ Object

Returns a Result with processed: { “verdict” => @verdict } so the pipeline can handle agents and tribunals through the same interface.



97
98
99
100
101
102
103
104
 
# File 'lib/active_harness/tribunal.rb', line 97

def result
  Result.new(
    input:          @input,
    output:         nil,
    processed:      { "verdict" => @verdict },
    execution_time: @execution_time
  )
end