Class: CompletionKit::PromptImprovementValidator

Inherits:
Object
  • Object
show all
Defined in:
app/services/completion_kit/prompt_improvement_validator.rb

Defined Under Namespace

Classes: Candidate

Constant Summary collapse

HELD_OUT_LIMIT = 
30

Instance Method Summary collapse

Constructor Details

#initialize(run, candidate_template, generator: nil, judge: nil) ⇒ PromptImprovementValidator

Returns a new instance of PromptImprovementValidator.



9
10
11
12
13
14
 
# File 'app/services/completion_kit/prompt_improvement_validator.rb', line 9

def initialize(run, candidate_template, generator: nil, judge: nil)
  @run = run
  @candidate = candidate_template
  @generator = generator || method(:generate)
  @judge = judge || method(:judge_score)
end

Instance Method Details

#call ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'app/services/completion_kit/prompt_improvement_validator.rb', line 16

def call
  rows = held_out.filter_map do |response|
    new_text = @generator.call(response)
    next if new_text.blank?

    after = @judge.call(response, new_text)
    next if after.nil?

    row_for(response, after)
  rescue StandardError
    next
  end
  summarize(rows, @total.to_i, @total.to_i > HELD_OUT_LIMIT)
end