Module: Legion::LLM::Inference::Steps::Debate

Includes:
Logging, Legion::Logging::Helper
Included in:
Executor
Defined in:
lib/legion/llm/inference/steps/debate.rb

Constant Summary collapse

CHALLENGER_PROMPT = 
<<~PROMPT
  You are a critical analyst reviewing the following response. Your job is to identify
  weaknesses, logical flaws, unsupported assumptions, missing context, or alternative
  perspectives that were not considered. Be specific and constructive.

  Original question/request:
  %<question>s

  Advocate's response:
  %<advocate>s

  Provide a thorough critique. What is wrong, incomplete, or could be improved?
PROMPT
REBUTTAL_PROMPT = 
<<~PROMPT
  You originally provided a response to a question. A challenger has critiqued your
  response. Address the critique directly, defending valid points and conceding where
  the challenger identified genuine weaknesses.

  Original question/request:
  %<question>s

  Your original response:
  %<advocate>s

  Challenger's critique:
  %<challenger>s

  Provide a rebuttal that strengthens your position or acknowledges valid criticisms.
PROMPT
JUDGE_PROMPT = 
<<~PROMPT
  You are an impartial judge evaluating a multi-round debate about the following question.
  Your task is to evaluate the strength of each position, state which argument was stronger
  and why, assign confidence to the final position, then produce the most accurate,
  balanced, and complete answer possible.

  Original question/request:
  %<question>s

  Advocate's position:
  %<advocate>s

  Challenger's critique:
  %<challenger>s

  Advocate's rebuttal:
  %<rebuttal>s

  Respond using exactly these section labels:
  Evaluation: compare the advocate and challenger, state which side was stronger, and give confidence.
  Final answer: synthesize the final answer. Incorporate valid points from the critique while preserving
  what the advocate got right. Be direct and definitive.
PROMPT

Instance Method Summary collapse

Instance Method Details

#debate_enabled?(request) ⇒ Boolean

Returns:

  • (Boolean) 


119
120
121
122
123
124
125
126
127
 
# File 'lib/legion/llm/inference/steps/debate.rb', line 119

def debate_enabled?(request)
  explicit = request.extra[:debate] if request.extra.is_a?(Hash)
  return explicit unless explicit.nil?

  gaia_trigger = gaia_debate_trigger?(@enrichments)
  return true if gaia_trigger

  settings_value(:debate, :enabled) == true
end

#gaia_debate_trigger?(enrichments) ⇒ Boolean

Returns:

  • (Boolean) 


129
130
131
132
133
134
135
136
137
 
# File 'lib/legion/llm/inference/steps/debate.rb', line 129

def gaia_debate_trigger?(enrichments)
  return false unless debate_setting(:gaia_auto_trigger) == true

  gaia = enrichments&.fetch('gaia:advisory', nil)
  advisory = gaia.is_a?(Hash) ? gaia[:data] : nil
  return false unless advisory.is_a?(Hash)

  advisory[:high_stakes] == true || advisory[:debate_recommended] == true
end

#run_debate(advocate_response, request) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/legion/llm/inference/steps/debate.rb', line 139

def run_debate(advocate_response, request)
  rounds      = resolve_debate_rounds(request)
  question    = extract_question(request)
  advocate_text = extract_content(advocate_response)

  models = select_debate_models(request)
  @warnings << models[:warning] if models[:warning]
  if models[:skip]
    log.warn(
      "[llm][steps][debate] action=skipped_insufficient_models request_id=#{request&.id || 'none'} " \
      "reason=#{models[:skip]} available_models=#{available_models.size}"
    )
    return nil
  end

  advocate_model    = models[:advocate]
  challenger_model  = models[:challenger]
  judge_model       = models[:judge]

  current_advocate = advocate_text
  log_debate_models(rounds, advocate_model, challenger_model, judge_model)
  current_challenger, current_rebuttal = run_debate_rounds(
    rounds: rounds, question: question, advocate_model: advocate_model,
    challenger_model: challenger_model, current_advocate: current_advocate
  )

  judge_synthesis = judge_debate(
    question:           question,
    advocate_text:      advocate_text,
    current_challenger: current_challenger,
    current_rebuttal:   current_rebuttal,
    judge_model:        judge_model
  )
  judge_sections = parse_judge_output(judge_synthesis)
  synthetic_response = SyntheticResponse.new(judge_sections[:final_answer])

  {
    synthetic_response: synthetic_response,
    rounds:             rounds,
    metadata:           debate_metadata(
      rounds: rounds, advocate_model: advocate_model, challenger_model: challenger_model,
      judge_model: judge_model, advocate_text: advocate_text,
      current_challenger: current_challenger, judge_sections: judge_sections,
      judge_synthesis: judge_synthesis
    )
  }
end

#step_debate ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/legion/llm/inference/steps/debate.rb', line 69

def step_debate
  unless debate_enabled?(@request)
    log_step_debug(:debate, :skipped, reason: :disabled)
    return
  end
  unless @raw_response
    log_step_debug(:debate, :skipped, reason: :no_response)
    return
  end

  log_step_info(:debate, :start, message_count: @request.messages.size)
  debate_result = run_debate(@raw_response, @request)
  unless debate_result
    log_step_debug(:debate, :skipped, reason: :no_result)
    return
  end

  original_chars = extract_content(@raw_response).length
  @raw_response = debate_result[:synthetic_response]
  log.warn(
    "[llm][steps][debate] action=response_replaced request_id=#{@request&.id || 'none'} " \
    "original_chars=#{original_chars} synthetic_chars=#{debate_result[:synthetic_response].content.to_s.length} " \
    "rounds=#{debate_result[:rounds]}"
  )
  @enrichments['debate:result'] = {
    content:   "debate completed: #{debate_result[:rounds]} rounds, judge synthesis produced",
    data:      debate_result[:metadata],
    timestamp: Time.now
  }

  @timeline.record(
    category: :internal, key: 'debate:completed',
    direction: :internal,
    detail: "rounds=#{debate_result[:rounds]} advocate=#{debate_result[:metadata][:advocate_model]} " \
            "challenger=#{debate_result[:metadata][:challenger_model]} judge=#{debate_result[:metadata][:judge_model]}",
    from: 'pipeline', to: 'pipeline'
  )
  log_step_info(
    :debate,
    :complete,
    rounds:           debate_result[:rounds],
    advocate_model:   debate_result[:metadata][:advocate_model],
    challenger_model: debate_result[:metadata][:challenger_model],
    judge_model:      debate_result[:metadata][:judge_model]
  )
rescue StandardError => e
  @warnings << "debate step error: #{e.message}"
  handle_exception(e, level: :warn, operation: 'llm.pipeline.steps.debate')
end