Module: Legion::LLM::Pipeline::Steps::ConfidenceScoring

Includes:
Legion::Logging::Helper
Included in:
Executor
Defined in:
lib/legion/llm/pipeline/steps/confidence_scoring.rb

Instance Method Summary collapse

Instance Method Details

#step_confidence_scoring ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/legion/llm/pipeline/steps/confidence_scoring.rb', line 12

def step_confidence_scoring
  return unless @raw_response

  opts = {
    json_expected:     @request.response_format&.dig(:type) == :json,
    quality_threshold: @request.extra&.dig(:quality_threshold),
    confidence_score:  @request.extra&.dig(:confidence_score),
    confidence_bands:  @request.extra&.dig(:confidence_bands)
  }.compact

  @confidence_score = ConfidenceScorer.score(@raw_response, **opts)

  @timeline.record(
    category: :internal, key: 'confidence:scored',
    direction: :internal,
    detail: "score=#{@confidence_score.score.round(3)} band=#{@confidence_score.band} source=#{@confidence_score.source}",
    from: 'pipeline', to: 'pipeline'
  )
rescue StandardError => e
  @warnings << "confidence_scoring error: #{e.message}"
  handle_exception(e, level: :warn, operation: 'llm.pipeline.steps.confidence_scoring')
  @confidence_score = nil
end