Module: Qualspec

Defined in:
lib/qualspec.rb,
lib/qualspec.rb,
lib/qualspec/judge.rb,
lib/qualspec/rspec.rb,
lib/qualspec/client.rb,
lib/qualspec/rubric.rb,
lib/qualspec/version.rb,
lib/qualspec/recorder.rb,
lib/qualspec/suite/dsl.rb,
lib/qualspec/evaluation.rb,
lib/qualspec/suite/runner.rb,
lib/qualspec/configuration.rb,
lib/qualspec/rspec/helpers.rb,
lib/qualspec/prompt_variant.rb,
lib/qualspec/rspec/matchers.rb,
lib/qualspec/suite/behavior.rb,
lib/qualspec/suite/reporter.rb,
lib/qualspec/suite/scenario.rb,
lib/qualspec/builtin_rubrics.rb,
lib/qualspec/suite/candidate.rb,
lib/qualspec/rspec/configuration.rb,
lib/qualspec/suite/html_reporter.rb,
lib/qualspec/rspec/evaluation_result.rb,
lib/qualspec/suite/builtin_behaviors.rb

Defined Under Namespace

Modules: BuiltinRubrics, RSpec, Suite Classes: Client, Configuration, Error, Evaluation, Judge, PromptVariant, Recorder, Rubric

Constant Summary collapse

VERSION = 
'0.1.2'

Class Method Summary collapse

Class Method Details

.client ⇒ Object 



45
46
47
 
# File 'lib/qualspec.rb', line 45

def client
  @client ||= Client.new(configuration)
end

.configuration ⇒ Object 



28
29
30
 
# File 'lib/qualspec.rb', line 28

def configuration
  @configuration ||= Configuration.new
end

.configure {|configuration| ... } ⇒ Object

Yields:



32
33
34
 
# File 'lib/qualspec.rb', line 32

def configure
  yield(configuration)
end

.define_behavior(name, &block) ⇒ Object

Top-level convenience method



59
60
61
 
# File 'lib/qualspec.rb', line 59

def define_behavior(name, &block)
  Suite::Behavior.define(name, &block)
end

.define_rubric(name, &block) ⇒ Object

Convenience method for defining rubrics



54
55
56
 
# File 'lib/qualspec.rb', line 54

def define_rubric(name, &block)
  Rubric.define(name, &block)
end

.evaluation(name, &block) ⇒ Object

Top-level convenience method



64
65
66
 
# File 'lib/qualspec.rb', line 64

def evaluation(name, &block)
  Suite.define(name, &block)
end

.judge ⇒ Object 



49
50
51
 
# File 'lib/qualspec.rb', line 49

def judge
  @judge ||= Judge.new
end

.reset! ⇒ Object 



36
37
38
39
40
41
42
43
 
# File 'lib/qualspec.rb', line 36

def reset!
  @configuration = nil
  @client = nil
  @judge = nil
  Rubric.clear!
  Suite.clear!
  Suite::Behavior.clear!
end

.run(suite_name, progress: true, output: :stdout, json_path: nil, html_path: nil, show_responses: false, load_builtins: true) ⇒ Object

Run an evaluation suite



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/qualspec.rb', line 69

def run(suite_name, progress: true, output: :stdout, json_path: nil, html_path: nil, show_responses: false,
        load_builtins: true)
  # Load builtins (idempotent, can be called multiple times)
  if load_builtins
    BuiltinRubrics.load!
    Suite::BuiltinBehaviors.load!
  end

  suite = Suite.find(suite_name)
  runner = Suite::Runner.new(suite)

  results = runner.run(progress: progress)
  results.finish!

  reporter = Suite::Reporter.new(results, show_responses: show_responses)

  case output
  when :stdout
    puts reporter.to_stdout
  when :json
    puts reporter.to_json
  when :silent
    # nothing
  end

  reporter.write_json(json_path) if json_path

  if html_path
    html_reporter = Suite::HtmlReporter.new(results)
    html_reporter.write(html_path)
  end

  results
end