Class: Synthra::QualityMetrics::Analyzer

Inherits:
Object
  • Object
show all
Defined in:
lib/synthra/quality_metrics.rb

Overview

Data quality analyzer

Instance Method Summary collapse

Constructor Details

#initialize(records, schema = nil) ⇒ Analyzer

Returns a new instance of Analyzer.



50
51
52
53
54
 
# File 'lib/synthra/quality_metrics.rb', line 50

def initialize(records, schema = nil)
  @records = records
  @schema = schema
  @field_names = detect_field_names
end

Instance Method Details

#analyze ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/synthra/quality_metrics.rb', line 56

def analyze
  QualityReport.new(
    total_records: @records.length,
    uniqueness: calculate_uniqueness,
    distribution: calculate_distribution,
    completeness: calculate_completeness,
    type_consistency: calculate_type_consistency,
    realism_score: calculate_realism_score,
    patterns: detect_patterns,
    anomalies: detect_anomalies
  )
end

#calculate_completeness ⇒ Object (private) 



112
113
114
115
116
117
118
119
120
121
 
# File 'lib/synthra/quality_metrics.rb', line 112

def calculate_completeness
  total_fields = @records.length * @field_names.length
  return 100.0 if total_fields.zero?

  filled_fields = @records.sum do |record|
    @field_names.count { |f| !record[f].nil? && record[f] != "" }
  end

  (filled_fields.to_f / total_fields * 100).round(2)
end

#calculate_distribution ⇒ Object (private) 



92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/synthra/quality_metrics.rb', line 92

def calculate_distribution
  result = {}

  @field_names.each do |field|
    values = @records.map { |r| r[field] }.compact

    # Only calculate distribution for fields with limited cardinality
    unique_values = values.uniq
    next if unique_values.length > 50 || unique_values.length == values.length

    distribution = values.tally.transform_values do |count|
      (count.to_f / values.length * 100).round(2)
    end

    result[field] = distribution
  end

  result
end

#calculate_realism_score ⇒ Object (private) 



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/synthra/quality_metrics.rb', line 141

def calculate_realism_score
  scores = []

  # Score based on uniqueness of ID fields
  id_fields = @field_names.select { |f| f.match?(/\A(id|uuid|_id)\z/i) }
  id_fields.each do |field|
    uniqueness = calculate_uniqueness[field] || 0
    scores << uniqueness
  end

  # Score based on email format
  email_fields = @field_names.select { |f| f.match?(/email/i) }
  email_fields.each do |field|
    values = @records.map { |r| r[field] }.compact
    valid_emails = values.count { |v| v.to_s.match?(/@.*\./) }
    scores << (valid_emails.to_f / values.length * 100) if values.any?
  end

  # Score based on completeness
  scores << calculate_completeness

  # Score based on type consistency
  type_scores = calculate_type_consistency.values
  scores << (type_scores.sum / type_scores.length) if type_scores.any?

  # :nocov: unreachable in practice — completeness above always appends one score, so `scores`
  # is never empty; kept as a defensive guard against future refactors.
  return 0.0 if scores.empty?
  # :nocov:

  (scores.sum / scores.length).round(2)
end

#calculate_type_consistency ⇒ Object (private) 



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/synthra/quality_metrics.rb', line 123

def calculate_type_consistency
  result = {}

  @field_names.each do |field|
    values = @records.map { |r| r[field] }.compact
    next if values.empty?

    types = values.map { |v| classify_type(v) }
    type_counts = types.tally

    # Consistency = % of most common type
    most_common_count = type_counts.values.max
    result[field] = (most_common_count.to_f / types.length * 100).round(2)
  end

  result
end

#calculate_uniqueness ⇒ Object (private) 



77
78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/synthra/quality_metrics.rb', line 77

def calculate_uniqueness
  result = {}

  @field_names.each do |field|
    values = @records.map { |r| r[field] }.compact
    next if values.empty?

    unique_count = values.uniq.length
    total_count = values.length
    result[field] = (unique_count.to_f / total_count * 100).round(2)
  end

  result
end

#classify_type(value) ⇒ Object (private) 



246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
 
# File 'lib/synthra/quality_metrics.rb', line 246

def classify_type(value)
  case value
  when nil then :null
  when Integer then :integer
  when Float then :float
  when TrueClass, FalseClass then :boolean
  when Array then :array
  when Hash then :object
  when String
    if value.match?(/\A\d{4}-\d{2}-\d{2}/)
      :datetime
    elsif value.match?(/\A[0-9a-f-]{36}\z/i)
      :uuid
    else
      :string
    end
  else
    :unknown
  end
end

#detect_anomalies ⇒ Object (private) 



198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
 
# File 'lib/synthra/quality_metrics.rb', line 198

def detect_anomalies
  anomalies = []

  # Check for unexpected null ratios
  @field_names.each do |field|
    values = @records.map { |r| r[field] }
    null_ratio = values.count(&:nil?).to_f / values.length

    if null_ratio > 0.5
      anomalies << {
        type: :high_null_ratio,
        field: field,
        ratio: (null_ratio * 100).round(2)
      }
    end
  end

  # Check for duplicate IDs
  id_fields = @field_names.select { |f| f.match?(/\A(id|uuid|_id)\z/i) }
  id_fields.each do |field|
    values = @records.map { |r| r[field] }.compact
    duplicates = values.length - values.uniq.length

    if duplicates > 0
      anomalies << {
        type: :duplicate_ids,
        field: field,
        count: duplicates
      }
    end
  end

  # Check for unrealistic distributions
  calculate_distribution.each do |field, dist|
    max_percentage = dist.values.max || 0
    if max_percentage > 90
      anomalies << {
        type: :skewed_distribution,
        field: field,
        dominant_value: dist.key(max_percentage),
        percentage: max_percentage
      }
    end
  end

  anomalies
end

#detect_field_names ⇒ Object (private) 



71
72
73
74
75
 
# File 'lib/synthra/quality_metrics.rb', line 71

def detect_field_names
  return @schema.fields.map(&:name) if @schema

  @records.first&.keys || []
end

#detect_patterns ⇒ Object (private) 



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
 
# File 'lib/synthra/quality_metrics.rb', line 174

def detect_patterns
  patterns = {}

  @field_names.each do |field|
    values = @records.map { |r| r[field] }.compact
    next if values.empty?

    string_values = values.select { |v| v.is_a?(String) }
    next if string_values.length < 10

    # Detect length patterns
    lengths = string_values.map(&:length)
    patterns[field] = {
      min_length: lengths.min,
      max_length: lengths.max,
      avg_length: (lengths.sum.to_f / lengths.length).round(2),
      common_prefixes: find_common_prefixes(string_values),
      common_suffixes: find_common_suffixes(string_values)
    }
  end

  patterns
end

#find_common_prefixes(strings, min_length: 2) ⇒ Object (private) 



267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
 
# File 'lib/synthra/quality_metrics.rb', line 267

def find_common_prefixes(strings, min_length: 2)
  return [] if strings.length < 10

  prefixes = Hash.new(0)
  strings.each do |s|
    (min_length..[s.length, 10].min).each do |len|
      prefixes[s[0, len]] += 1
    end
  end

  prefixes.select { |_, count| count > strings.length * 0.1 }
          .sort_by { |_, count| -count }
          .first(3)
          .map(&:first)
end

#find_common_suffixes(strings, min_length: 2) ⇒ Object (private) 



283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/synthra/quality_metrics.rb', line 283

def find_common_suffixes(strings, min_length: 2)
  return [] if strings.length < 10

  suffixes = Hash.new(0)
  strings.each do |s|
    (min_length..[s.length, 10].min).each do |len|
      suffixes[s[-len..]] += 1
    end
  end

  suffixes.select { |_, count| count > strings.length * 0.1 }
          .sort_by { |_, count| -count }
          .first(3)
          .map(&:first)
end