Class: CompletionKit::Metric

Inherits:
ApplicationRecord show all
Includes:
Taggable
Defined in:
app/models/completion_kit/metric.rb

Constant Summary collapse

DEFAULT_RUBRIC_BANDS = 
[
  { "stars" => 5, "description" => "Fully meets or exceeds all criteria. No meaningful issues." },
  { "stars" => 4, "description" => "Meets criteria well. Minor issues only." },
  { "stars" => 3, "description" => "Meets criteria adequately. Some room for improvement." },
  { "stars" => 2, "description" => "Partially meets criteria. Significant gaps or frequent errors." },
  { "stars" => 1, "description" => "Fails to meet the criteria. Major errors or completely off-target." }
].freeze
METRIC_TYPES = 
%w[llm_judge check].freeze

Constants inherited from ApplicationRecord

ApplicationRecord::TenantScopedUniquenessValidator

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Taggable

#tag_names, #tag_names=

Class Method Details

.default_rubric_bands ⇒ Object 



36
37
38
 
# File 'app/models/completion_kit/metric.rb', line 36

def self.default_rubric_bands
  DEFAULT_RUBRIC_BANDS.map(&:dup)
end

.default_rubric_text ⇒ Object 



40
41
42
 
# File 'app/models/completion_kit/metric.rb', line 40

def self.default_rubric_text
  rubric_text_for(default_rubric_bands)
end

.normalize_rubric_bands(raw_bands) ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'app/models/completion_kit/metric.rb', line 52

def self.normalize_rubric_bands(raw_bands)
  bands = raw_bands.is_a?(Hash) ? raw_bands.values : Array(raw_bands)
  band_map = bands.each_with_object({}) do |band, acc|
    next unless band.respond_to?(:to_h)

    normalized = band.to_h.stringify_keys.slice("stars", "description")
    stars = normalized["stars"].to_i
    next unless (1..5).cover?(stars)

    acc[stars] = {
      "stars" => stars,
      "description" => normalized["description"].to_s.strip
    }
  end

  default_rubric_bands.map do |default_band|
    stars = default_band["stars"]
    band = band_map[stars]
    {
      "stars" => stars,
      "description" => band && band["description"].present? ? band["description"] : default_band["description"]
    }
  end
end

.rubric_text_for(bands) ⇒ Object 



44
45
46
47
48
49
50
 
# File 'app/models/completion_kit/metric.rb', line 44

def self.rubric_text_for(bands)
  Array(bands).sort_by { |b| -(b["stars"] || 0) }.map do |band|
    stars = band["stars"].to_i
    label = stars == 1 ? "1 star" : "#{stars} stars"
    "#{label}: #{band["description"]}"
  end.join("\n\n")
end

Instance Method Details

#as_json(options = {}) ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'app/models/completion_kit/metric.rb', line 97

def as_json(options = {})
  base = {
    id: id, name: name, key: key, metric_type: metric_type,
    created_at: created_at, updated_at: updated_at,
    tags: tags.as_json
  }
  if check?
    base.merge(check_config: check_config)
  else
    base.merge(instruction: instruction, rubric_bands: rubric_bands)
  end
end

#check? ⇒ Boolean

Returns:

  • (Boolean) 


85
86
87
 
# File 'app/models/completion_kit/metric.rb', line 85

def check?
  metric_type == "check"
end

#display_rubric_text ⇒ Object 



81
82
83
 
# File 'app/models/completion_kit/metric.rb', line 81

def display_rubric_text
  self.class.rubric_text_for(rubric_bands_for_form)
end

#in_use? ⇒ Boolean

Returns:

  • (Boolean) 


93
94
95
 
# File 'app/models/completion_kit/metric.rb', line 93

def in_use?
  RunMetric.exists?(metric_id: id) || reviews.exists? || metric_versions.exists?
end

#llm_judge? ⇒ Boolean

Returns:

  • (Boolean) 


89
90
91
 
# File 'app/models/completion_kit/metric.rb', line 89

def llm_judge?
  !check?
end

#rubric_bands_for_form ⇒ Object 



77
78
79
 
# File 'app/models/completion_kit/metric.rb', line 77

def rubric_bands_for_form
  self.class.normalize_rubric_bands(rubric_bands)
end