Class: CompletionKit::MetricsController

Inherits:
ApplicationController show all
Includes:
TagFiltering
Defined in:
app/controllers/completion_kit/metrics_controller.rb

Constant Summary

Constants inherited from ApplicationController

ApplicationController::ONBOARDING_DISMISS_COOKIE

Instance Method Summary collapse

Instance Method Details

#add_few_shot ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 128

def add_few_shot
  calibration = Calibration.where(metric_id: @metric.id, verdict: "disagree").find(params[:calibration_id])
  review = calibration.response.reviews.find_by(metric_id: @metric.id)
  examples = Array(@metric.few_shot_examples)
  examples << {
    "input" => calibration.response.input_data.to_s.truncate(2000),
    "response" => calibration.response.response_text.to_s.truncate(2000),
    "judge_score" => review&.ai_score&.to_f,
    "judge_feedback" => review&.ai_feedback.to_s.truncate(1000),
    "human_score" => calibration.corrected_score&.to_f,
    "human_note" => calibration.note.to_s.truncate(1000),
    "calibration_id" => calibration.id,
    "added_at" => Time.current.utc.iso8601
  }
  @metric.update!(few_shot_examples: examples)
  redirect_to metric_path(@metric), notice: "Got it. The judge will remember this next time it grades."
end

#adopt_starter ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 16

def adopt_starter
  starter = StarterMetrics.find(params[:key])
  return redirect_to(metrics_path, alert: "Unknown starter metric.") unless starter
  if Metric.exists?(name: starter.name)
    return redirect_to(metrics_path, alert: "A metric named \"#{starter.name}\" already exists.")
  end
  metric = Metric.create!(
    name: starter.name,
    instruction: starter.instruction,
    rubric_bands: starter.rubric_bands
  )
  redirect_to metric_path(metric), notice: "Added the \"#{starter.name}\" starter. Tweak any band before you run a judge against it."
end

#create ⇒ Object 



60
61
62
63
64
65
66
67
68
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 60

def create
  @metric = Metric.new(metric_params)

  if @metric.save
    redirect_to metric_path(@metric), notice: "Metric was successfully created."
  else
    render :new, status: :unprocessable_entity
  end
end

#destroy ⇒ Object 



78
79
80
81
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 78

def destroy
  @metric.destroy
  redirect_to metrics_path, notice: "Metric was successfully destroyed."
end

#dismiss_starter ⇒ Object 



30
31
32
33
34
35
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 30

def dismiss_starter
  starter = StarterMetrics.find(params[:key])
  return redirect_to(metrics_path, alert: "Unknown starter metric.") unless starter
  StarterMetricDismissal.find_or_create_by(starter_key: starter.key)
  redirect_to metrics_path, notice: "Dismissed \"#{starter.name}\". It won't appear here again."
end

#dismiss_suggestion ⇒ Object 



103
104
105
106
107
108
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 103

def dismiss_suggestion
  draft = JudgeVersion.drafts.where(metric_id: @metric.id).find_by(id: params[:draft_id])
  draft&.destroy
  target = params[:back_to] == "edit" ? edit_metric_path(@metric) : metric_path(@metric)
  redirect_to target, notice: "Dismissed."
end

#edit ⇒ Object 



54
55
56
57
58
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 54

def edit
  @suggestion_draft = JudgeVersion.drafts.where(metric_id: @metric.id, source: "suggestion").order(created_at: :desc).first
  @edit_draft = JudgeVersion.drafts.where(metric_id: @metric.id, source: "edit").order(created_at: :desc).first
  @published_judge_version = JudgeVersion.published.where(metric_id: @metric.id, current: true).first
end

#index ⇒ Object 



6
7
8
9
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 6

def index
  @metrics = apply_tag_filter(Metric.includes(:metric_groups, :tags).order(:name))
  @available_starters = StarterMetrics.available
end

#new ⇒ Object 



50
51
52
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 50

def new
  @metric = Metric.new
end

#publish_draft ⇒ Object 



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 110

def publish_draft
  scope = JudgeVersion.where(metric_id: @metric.id)
  version = if params[:draft_id].present?
              scope.find_by(id: params[:draft_id])
            else
              JudgeVersion.drafts.where(metric_id: @metric.id).order(created_at: :desc).first
            end

  if version.nil?
    redirect_to metric_path(@metric), alert: "No version to publish."
    return
  end

  version.publish!
  redirect_to metric_path(@metric),
              notice: "#{@metric.name} #{version.version_label} is now the published version."
end

#remove_few_shot ⇒ Object 



146
147
148
149
150
151
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 146

def remove_few_shot
  cal_id = params[:calibration_id].to_i
  remaining = Array(@metric.few_shot_examples).reject { |fs| fs["calibration_id"].to_i == cal_id }
  @metric.update!(few_shot_examples: remaining)
  redirect_to metric_path(@metric), notice: "Forgotten. The judge won't see this case next time."
end

#show ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 37

def show
  @published_judge_version = JudgeVersion.ensure_current_for(@metric)
  @disagreements = Calibration.where(metric_id: @metric.id, verdict: "disagree")
                              .includes(:judge_version, response: [:reviews, :run])
                              .order(created_at: :desc)
                              .limit(50)
  @edit_draft = JudgeVersion.drafts.where(metric_id: @metric.id, source: "edit").order(created_at: :desc).first
  @suggestion_draft = JudgeVersion.drafts.where(metric_id: @metric.id, source: "suggestion").order(created_at: :desc).first
  @improve_disagreement_count = Calibration.where(metric_id: @metric.id, verdict: "disagree",
                                                  judge_version_id: @published_judge_version.id).count
  @versions = JudgeVersion.where(metric_id: @metric.id).order(version_number: :desc).to_a
end

#starter_preview ⇒ Object 



11
12
13
14
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 11

def starter_preview
  @starter = StarterMetrics.find(params[:key])
  return redirect_to(metrics_path, alert: "Unknown starter metric.") unless @starter
end

#suggest_variants ⇒ Object 



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 83

def suggest_variants
  target = params[:back_to] == "edit" ? edit_metric_path(@metric) : metric_path(@metric)
  disagreement_count = Calibration.where(metric_id: @metric.id, verdict: "disagree").count
  if disagreement_count.zero?
    redirect_to target, alert: "Mark at least one row as Disagree before asking the model to suggest a change."
    return
  end

  JudgeVersion.drafts.where(metric_id: @metric.id, source: "suggestion").destroy_all

  generator = JudgeVariantGenerator.new(@metric, count: 1)
  variants = generator.call
  if variants.empty?
    redirect_to target, alert: "The model returned no usable variants. Try again with a different model."
    return
  end
  generator.persist!(variants)
  redirect_to target, notice: "Drafted a new version. Review it below."
end

#update ⇒ Object 



70
71
72
73
74
75
76
 
# File 'app/controllers/completion_kit/metrics_controller.rb', line 70

def update
  if @metric.update(metric_params)
    redirect_to metric_path(@metric), notice: "Metric was successfully updated."
  else
    render :edit, status: :unprocessable_entity
  end
end