Class: AISpec::Core::Statistics::Calculator

Inherits:
Object
  • Object
show all
Defined in:
lib/aispec/core/statistics/calculator.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(results) ⇒ Calculator

Returns a new instance of Calculator.



9
10
11
 
# File 'lib/aispec/core/statistics/calculator.rb', line 9

def initialize(results)
  @results = results || []
end

Instance Attribute Details

#results ⇒ Object (readonly)

Returns the value of attribute results.



7
8
9
 
# File 'lib/aispec/core/statistics/calculator.rb', line 7

def results
  @results
end

Instance Method Details

#average_judge_score ⇒ Object 



66
67
68
69
70
 
# File 'lib/aispec/core/statistics/calculator.rb', line 66

def average_judge_score
  scores = judge_scores
  return nil if scores.empty?
  (scores.sum / scores.size).round(2)
end

#confidence_interval ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/aispec/core/statistics/calculator.rb', line 34

def confidence_interval
  return 0.0 if total_assertions.zero?

  # Wilson score interval z = 1.96 for 95% confidence
  p = passed_assertions.to_f / total_assertions
  n = total_assertions.to_f
  z = 1.96

  margin = z * Math.sqrt((p * (1 - p) + (z**2 / (4 * n))) / n) / (1 + (z**2 / n))
  (margin * 100.0).round(1)
end

#failed_assertions ⇒ Object 



25
26
27
 
# File 'lib/aispec/core/statistics/calculator.rb', line 25

def failed_assertions
  total_assertions - passed_assertions
end

#judge_scores ⇒ Object 



60
61
62
63
64
 
# File 'lib/aispec/core/statistics/calculator.rb', line 60

def judge_scores
  @results.flat_map do |r|
    r[:assertion_results].select { |ar| ar.name == :judge && ar.metadata[:score] }.map { |ar| ar.metadata[:score].to_f }
  end
end

#mean_cost ⇒ Object 



51
52
53
54
 
# File 'lib/aispec/core/statistics/calculator.rb', line 51

def mean_cost
  return 0.0 if @results.empty?
  @results.sum { |r| r[:cost].to_f } / @results.size
end

#mean_latency ⇒ Object 



46
47
48
49
 
# File 'lib/aispec/core/statistics/calculator.rb', line 46

def mean_latency
  return 0.0 if @results.empty?
  (@results.sum { |r| r[:latency_ms].to_f } / @results.size).round(1)
end

#passed_assertions ⇒ Object 



21
22
23
 
# File 'lib/aispec/core/statistics/calculator.rb', line 21

def passed_assertions
  @results.sum { |r| r[:assertion_results].count(&:passed?) }
end

#recommendation ⇒ Object 



88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/aispec/core/statistics/calculator.rb', line 88

def recommendation
  rate = success_rate
  var = variance_rating

  if rate >= 95.0 && var != "High"
    "Stable"
  elsif rate >= 80.0 || var == "Moderate"
    "Needs Attention (Flaky)"
  else
    "Critical Failure"
  end
end

#success_rate ⇒ Object 



29
30
31
32
 
# File 'lib/aispec/core/statistics/calculator.rb', line 29

def success_rate
  return 0.0 if total_assertions.zero?
  (passed_assertions.to_f / total_assertions * 100.0).round(1)
end

#summary ⇒ Object 



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/aispec/core/statistics/calculator.rb', line 101

def summary
  {
    total_runs: total_runs,
    total_assertions: total_assertions,
    passed_assertions: passed_assertions,
    failed_assertions: failed_assertions,
    success_rate: success_rate,
    confidence_interval: confidence_interval,
    mean_latency: mean_latency,
    mean_cost: mean_cost,
    total_cost: total_cost,
    average_judge_score: average_judge_score,
    variance: variance_rating,
    recommendation: recommendation
  }
end

#total_assertions ⇒ Object 



17
18
19
 
# File 'lib/aispec/core/statistics/calculator.rb', line 17

def total_assertions
  @results.sum { |r| r[:assertion_results].size }
end

#total_cost ⇒ Object 



56
57
58
 
# File 'lib/aispec/core/statistics/calculator.rb', line 56

def total_cost
  @results.sum { |r| r[:cost].to_f }
end

#total_runs ⇒ Object 



13
14
15
 
# File 'lib/aispec/core/statistics/calculator.rb', line 13

def total_runs
  @results.size
end

#variance_rating ⇒ Object 



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/aispec/core/statistics/calculator.rb', line 72

def variance_rating
  scores = judge_scores
  return "N/A" if scores.size < 2

  mean = scores.sum / scores.size
  var = scores.sum { |s| (s - mean)**2 } / scores.size

  if var < 0.01
    "Low"
  elsif var < 0.05
    "Moderate"
  else
    "High"
  end
end