Class: Inferno::TestRunner

Inherits:
Object
  • Object
show all
Includes:
Utils::MarkdownFormatter
Defined in:
lib/inferno/test_runner.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Utils::MarkdownFormatter

#format_markdown

Constructor Details

#initialize(test_session:, test_run:, resume: false) ⇒ TestRunner

Returns a new instance of TestRunner.



10
11
12
13
14
 
# File 'lib/inferno/test_runner.rb', line 10

def initialize(test_session:, test_run:, resume: false)
  @test_session = test_session
  @test_run = test_run
  @resuming = resume
end

Instance Attribute Details

#resuming ⇒ Object (readonly)

Returns the value of attribute resuming.



8
9
10
 
# File 'lib/inferno/test_runner.rb', line 8

def resuming
  @resuming
end

#test_run ⇒ Object (readonly)

Returns the value of attribute test_run.



8
9
10
 
# File 'lib/inferno/test_runner.rb', line 8

def test_run
  @test_run
end

#test_session ⇒ Object (readonly)

Returns the value of attribute test_session.



8
9
10
 
# File 'lib/inferno/test_runner.rb', line 8

def test_session
  @test_session
end

Instance Method Details

#around_test(_test) { ... } ⇒ Object

Wraps the execution of a single test. Prepend a module overriding this method to run instrumentation around each test, for example to emit a distinct trace, span, or timing metric per test instead of one that spans the whole run.

An override must call super exactly once and return its value unchanged: it is the test's result, and parent results are rolled up from it. An override that does not call super silently skips the test.

Parameters:

  • test (Class) —

    the test being run (an Inferno::Entities::Test subclass)

Yields:

  • executes the test and returns its result

Returns:

  • the value returned by the block



120
121
122
 
# File 'lib/inferno/test_runner.rb', line 120

def around_test(_test)
  yield
end

#check_inputs(test, _test_instance, inputs) ⇒ Object 



124
125
126
127
128
129
130
131
132
 
# File 'lib/inferno/test_runner.rb', line 124

def check_inputs(test, _test_instance, inputs)
  inputs.each do |key, value|
    optional = test.config.input_optional?(key)
    if value.nil? && !optional
      raise Exceptions::SkipException,
            "Input '#{test.config.input_name(key)}' is nil, skipping test."
    end
  end
end

#evaluate_runnable_result(runnable, runnable_instance, inputs = nil) ⇒ Object 



201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/inferno/test_runner.rb', line 201

def evaluate_runnable_result(runnable, runnable_instance, inputs = nil)
  return if !(runnable < Entities::Test) && !runnable.block

  if runnable < Entities::Test
    raise Exceptions::CancelException, 'Test cancelled by user' if test_run_is_cancelling

    check_inputs(runnable, runnable_instance, inputs)

    runnable_instance.load_named_requests
  end
  runnable_instance.instance_eval(&runnable.block)
  'pass'
rescue Exceptions::TestResultException => e
  runnable_instance.result_message = format_markdown(e.message)
  e.result
rescue StandardError => e
  Application['logger'].error(e.full_message)
  runnable_instance.result_message = format_markdown("Error: #{e.message}\n\n#{e.backtrace.first}")
  'error'
end

#existing_test_result(runnable) ⇒ Object 



57
58
59
 
# File 'lib/inferno/test_runner.rb', line 57

def existing_test_result(runnable)
  results_repo.result_for_test_run(runnable.reference_hash.merge(test_run_id: test_run.id))
end

#inputs_as_json(runnable, input_values) ⇒ Object 



244
245
246
247
248
249
250
251
252
253
 
# File 'lib/inferno/test_runner.rb', line 244

def inputs_as_json(runnable, input_values)
  inputs_array = runnable.inputs.map do |input_identifier|
    {
      name: runnable.config.input_name(input_identifier),
      value: input_values[input_identifier],
      type: runnable.config.input_type(input_identifier)
    }
  end
  JSON.generate(inputs_array)
end

#load_inputs(runnable) ⇒ Object 



235
236
237
238
239
240
241
242
 
# File 'lib/inferno/test_runner.rb', line 235

def load_inputs(runnable)
  runnable.inputs.each_with_object({}) do |input_identifier, input_hash|
    input_alias = runnable.config.input_name(input_identifier)
    input_type = runnable.config.input_type(input_identifier)
    input_hash[input_identifier] =
      session_data_repo.load(test_session_id: test_session.id, name: input_alias, type: input_type)
  end
end

#need_to_update_parent_result?(children, child_results) ⇒ Boolean

Determines if the parent result needs to be updated based on the results of its children.

The parent result needs to be updated if:

  • No custom result block is provided and all required children have corresponding required results.
  • A custom result block is provided and all children have corresponding results.

Returns:

  • (Boolean) 


227
228
229
230
231
232
233
 
# File 'lib/inferno/test_runner.rb', line 227

def need_to_update_parent_result?(children, child_results, &)
  required_children = children.select(&:required?)
  required_results = child_results.select(&:required?)

  (!block_given? && required_children.length == required_results.length) ||
    (block_given? && children.length == child_results.length)
end

#persist_result(params) ⇒ Object 



273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
 
# File 'lib/inferno/test_runner.rb', line 273

def persist_result(params)
  # A single result can cascade into many separate inserts (messages,
  # requests, headers, tags). Wrapping them in one transaction turns that
  # into a single lock acquisition instead of one per insert, cutting
  # down on sqlite write-lock contention with concurrent readers/writers.
  #
  # If you change this, run `bundle exec rake db:check_concurrency` to verify
  # SQLITE_BUSY is still avoided under concurrent test-run writes and status
  # polling (see lib/inferno/utils/db_concurrency_check.rb for why that's a
  # rake task and not a spec).
  result = Inferno::Application['db.connection'].transaction do
    results_repo.create(
      params.merge(test_run_id: test_run.id, test_session_id: test_session.id)
    )
  end

  run_results[result.runnable.id] = result
end

#results_repo ⇒ Object 



20
21
22
 
# File 'lib/inferno/test_runner.rb', line 20

def results_repo
  @results_repo ||= Repositories::Results.new
end

#roll_up_result(results) ⇒ Object 



292
293
294
 
# File 'lib/inferno/test_runner.rb', line 292

def roll_up_result(results)
  ResultSummarizer.new(results).summarize
end

#run(runnable, scratch = {}) ⇒ Object 



47
48
49
50
51
52
53
54
55
 
# File 'lib/inferno/test_runner.rb', line 47

def run(runnable, scratch = {})
  if runnable < Entities::Test
    return existing_test_result(runnable) || run_test(runnable, scratch) if resuming

    run_test(runnable, scratch)
  else
    run_group(runnable, scratch)
  end
end

#run_group(group, scratch) ⇒ Object 



134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/inferno/test_runner.rb', line 134

def run_group(group, scratch)
  group_inputs_with_values = group.available_inputs.map do |_input_identifier, input|
    {
      name: input.name,
      label: input.title,
      description: input.description,
      value: session_data_repo.load(test_session_id: test_session.id, name: input.name, type: input.type),
      type: input.type
    }
  end

  if group.children.empty?
    group_result = persist_result(group.reference_hash.merge(result: 'omit',
                                                             result_message: 'No tests defined',
                                                             input_json: JSON.generate(group_inputs_with_values),
                                                             output_json: '[]'))
    update_parent_result(group.parent)
    return group_result
  end

  group_instance = group.new

  group.children(test_session.suite_options).each do |child|
    result = run(child, scratch)
    group_instance.results << result
    break if result.waiting?
  end

  result = evaluate_runnable_result(group, group_instance) || roll_up_result(group_instance.results)

  group_result = persist_result(group.reference_hash.merge(
                                  messages: group_instance.messages,
                                  result:,
                                  result_message: group_instance.result_message,
                                  input_json: JSON.generate(group_inputs_with_values)
                                ))

  update_parent_result(group.parent)

  group_result
end

#run_results ⇒ Object 



16
17
18
 
# File 'lib/inferno/test_runner.rb', line 16

def run_results
  @run_results ||= {}
end

#run_test(test, scratch) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/inferno/test_runner.rb', line 61

def run_test(test, scratch)
  around_test(test) do
    inputs = load_inputs(test)
    input_json_string = inputs_as_json(test, inputs)

    test_instance =
      test.new(
        inputs:,
        test_session_id: test_session.id,
        scratch:,
        suite_options: test_session.suite_options_hash
      )

    result = evaluate_runnable_result(test, test_instance, inputs)

    outputs = save_outputs(test_instance)
    output_json_string = JSON.generate(outputs)

    result_params = {
      messages: test_instance.messages,
      requests: test_instance.requests,
      result:,
      result_message: test_instance.result_message,
      input_json: input_json_string,
      output_json: output_json_string
    }.merge(test.reference_hash)

    test_result =
      if result == 'wait'
        # The waiting status and the wait result must become visible to
        # readers atomically, so that pollers never see the run waiting
        # without the waiting test's result being present.
        Inferno::Application['db.connection'].transaction do
          test_runs_repo.mark_as_waiting(test_run.id, test_instance.identifier, test_instance.wait_timeout)
          persist_result(result_params)
        end
      else
        persist_result(result_params)
      end

    # If running a single test, update its parents' results. If running a
    # group or suite, #run_group handles updating the parents.
    update_parent_result(test.parent) if test_run.test_id.present?

    test_result
  end
end

#save_outputs(runnable_instance) ⇒ Object 



255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
 
# File 'lib/inferno/test_runner.rb', line 255

def save_outputs(runnable_instance)
  outputs =
    runnable_instance.outputs_to_persist.map do |output_identifier, value|
      output_name = runnable_instance.class.config.output_name(output_identifier)
      output_type = runnable_instance.class.config.output_type(output_identifier)
      {
        name: output_name,
        type: output_type,
        value: value.to_s
      }
    end

  outputs.compact!
  outputs.each do |output|
    session_data_repo.save(output.merge(test_session_id: test_session.id))
  end
end

#session_data_repo ⇒ Object 



28
29
30
 
# File 'lib/inferno/test_runner.rb', line 28

def session_data_repo
  @session_data_repo ||= Repositories::SessionData.new
end

#start ⇒ Object 



37
38
39
40
41
42
43
44
45
 
# File 'lib/inferno/test_runner.rb', line 37

def start
  test_runs_repo.mark_as_running(test_run.id) unless test_run.status == 'cancelling'

  run(test_run.runnable)

  test_runs_repo.mark_as_done(test_run.id) unless run_results.values.any?(&:waiting?)

  run_results.values
end

#test_run_is_cancelling ⇒ Object 



32
33
34
35
 
# File 'lib/inferno/test_runner.rb', line 32

def test_run_is_cancelling
  # forces db refetch of the test run status in case it is being cancelled
  test_runs_repo.status_for_test_run(test_run.id) == 'cancelling'
end

#test_runs_repo ⇒ Object 



24
25
26
 
# File 'lib/inferno/test_runner.rb', line 24

def test_runs_repo
  @test_runs_repo ||= Repositories::TestRuns.new
end

#update_parent_result(parent) ⇒ Object 



176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/inferno/test_runner.rb', line 176

def update_parent_result(parent)
  return if parent.nil?

  children = parent.children(test_session.suite_options)
  child_results = results_repo.current_results_for_test_session_and_runnables(test_session.id, children)
  return unless need_to_update_parent_result?(children, child_results, &parent.block)

  parent_instance = parent.new
  parent_instance.results << child_results
  old_result = results_repo.current_result_for_test_session(test_session.id, parent.reference_hash)&.result
  new_result = evaluate_runnable_result(parent, parent_instance) || roll_up_result(child_results)

  if new_result != old_result
    persist_result(parent.reference_hash.merge(
                     result: new_result,
                     result_message: parent_instance.result_message,
                     messages: parent_instance.messages
                   ))

    update_parent_result(parent.parent)
  end

  new_result
end