Class: RSpec::FlakeClassifier::Evaluation

Inherits:
Object
  • Object
show all
Defined in:
lib/rspec/flake/classifier/evaluation.rb,
sig/rspec/flake/classifier.rbs

Instance Method Summary collapse

Instance Method Details

#classification(predictions:, ground_truth:) ⇒ Hash[String, untyped]

Parameters:

  • predictions: (Array[Hash[String | Symbol, untyped]])
  • ground_truth: (Array[Hash[String | Symbol, untyped]])

Returns:

  • (Hash[String, untyped]) 


6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/rspec/flake/classifier/evaluation.rb', line 6

def classification(predictions:, ground_truth:)
  expected = index_by_id(ground_truth)
  rows = Array(predictions).filter_map do |prediction|
    id = record_id(prediction)
    truth = Array(expected[id])
    next if id.nil? || truth.empty?

    predicted = labels_for(prediction)
    {
      "id" => id,
      "truth" => truth,
      "predicted" => predicted,
      "top1" => truth.include?(predicted.first),
      "top2" => !(truth & predicted.first(2)).empty?
    }
  end
  metric_hash(rows)
end

#deflaker(records:) ⇒ Hash[String, untyped]

Parameters:

  • records: (Array[Hash[String | Symbol, untyped]])

Returns:

  • (Hash[String, untyped]) 


25
26
27
28
29
30
31
32
33
 
# File 'lib/rspec/flake/classifier/evaluation.rb', line 25

def deflaker(records:)
  flaky = Array(records).select { |record| flaky?(record) }
  captured = flaky.select { |record| deflaker_suspected?(record) }
  {
    "flaky_count" => flaky.length,
    "captured_count" => captured.length,
    "recall" => ratio(captured.length, flaky.length)
  }
end

#idflakies(records:) ⇒ Hash[String, untyped]

Parameters:

  • records: (Array[Hash[String | Symbol, untyped]])

Returns:

  • (Hash[String, untyped]) 


49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/rspec/flake/classifier/evaluation.rb', line 49

def idflakies(records:)
  counts = Hash.new(0)
  Array(records).each do |record|
    order_type = record["order_type"] || record[:order_type] || record.dig("investigation", "order_type")
    counts[order_type || "unknown"] += 1
  end
  total = counts.values.sum
  {
    "total" => total,
    "counts" => counts.sort.to_h,
    "od_ratio" => ratio(counts["od"], total),
    "nod_ratio" => ratio(counts["nod"], total)
  }
end

#signatures(records:) ⇒ Hash[String, untyped]

Parameters:

  • records: (Array[Hash[String | Symbol, untyped]])

Returns:

  • (Hash[String, untyped]) 


35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/rspec/flake/classifier/evaluation.rb', line 35

def signatures(records:)
  flaky = Array(records).select { |record| flaky?(record) }
  signed = flaky.select { |record| signature_for(record) }
  collision_groups = signed.group_by { |record| signature_for(record) }.values.count do |group|
    group.flat_map { |record| labels_for(record) }.uniq.length > 1
  end
  {
    "flaky_count" => flaky.length,
    "signed_count" => signed.length,
    "capture_rate" => ratio(signed.length, flaky.length),
    "collision_groups" => collision_groups
  }
end