Class: GeneratorTestResultsBacktrace

Inherits:
Object
  • Object
show all
Defined in:
lib/ceedling/generators/generator_test_results_backtrace.rb

Overview

=========================================================================

Ceedling - Test-Centered Build System for C ThrowTheSwitch.org Copyright (c) 2010-26 Mike Karlesky, Mark VanderVoord, & Greg Williams SPDX-License-Identifier: MIT

Instance Method Summary collapse

Instance Method Details

#do_gdb(filename, executable, shell_result, test_cases, context:) ⇒ Object

Re-runs each test case (or, for a parameterized test, each group of parameterized cases -- see group_test_cases) under gdb to identify which one(s) crashed and why. Writes the full gdb transcript to a per-test-case log file and assembles a terse crash label (signal + description, optional source line in backticks) for each failing test case. Returns a modified shell_result with regenerated output.



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/ceedling/generators/generator_test_results_backtrace.rb', line 24

def do_gdb(filename, executable, shell_result, test_cases, context:)
  gdb_script_filepath = File.join( @configurator.project_build_tests_root, BACKTRACE_GDB_SCRIPT_FILE )

  # Clean stats tracker
  test_case_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )

  # Reset time
  shell_result[:time] = 0

  test_name = File.basename( filename, '.*' )

  # True once some retry group has actually shown crash evidence of its own -- an
  # unresolved member, or (below) a group whose real status contradicts a fully clean
  # set of matches. If this stays false across every group, the whole diagnostic never
  # reproduced or attributed the crash the main run already detected, and none of it
  # can be trusted -- see the fallback after the loop.
  any_group_crashed = false

  # Iterate on test cases, one sub-process run per group (see `group_test_cases`)
  group_test_cases( test_cases ).each do |group|
    # Build the test fixture to run with our test case (or parameterized group) of interest
    command = @tool_executor.build_command_line(
      @configurator.tools_test_backtrace_gdb, [],
      gdb_script_filepath,
      executable,
      unity_filter_arg( group )
    )
    # Things are gonna go boom, so ignore booms to get output
    command[:options][:boom] = false

    crash_result = @tool_executor.exec( command )

    # Sum execution time for each sub-process run
    # Note: Running tests separately increases total execution time
    shell_result[:time] += crash_result[:time].to_f()

    # Buffered separately from test_case_results and only merged in afterward, since
    # the status check below can still discard every match here in favor of a crash
    # attribution, once the whole group has been seen.
    group_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
    unresolved = []

    # Attribute each group member its own real result line, if Unity printed one
    group.each do |test_case|
      case crash_result[:output]
      # Success test case
      when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:PASS\s*$)/
        group_results[:passed]  += 1
        group_results[:output] << $1

      # Ignored test case
      when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:IGNORE\s*$)/
        group_results[:ignored] += 1
        group_results[:output] << $1

      when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:FAIL(:.+)?\s*$)/
        group_results[:failed]  += 1
        group_results[:output] << $1

      # No result line for this member -- either it crashed, or it never got to run
      # because an earlier member in this same group crashed. Resolved below.
      else
        unresolved << test_case
      end
    end

    if !unresolved.empty?
      # Prefer whichever unresolved member's own C symbol is actually named in the gdb
      # backtrace (works regardless of position in the group); fall back to the first
      # unresolved member if no member's symbol can be found in the transcript (e.g. a
      # brief crash report with no frame information at all).
      crashed_case = unresolved.find do |tc|
        crash_result[:output].match?( /#{Regexp.escape(tc[:symbol])}\s*\(\)\sat/ )
      end
      crashed_case ||= unresolved.first

      # Per-test-case log file: <log_path>/<context>/<test_name>/<test_case>.gdb.log
      log_path = @file_path_utils.form_test_gdb_log( test_name, context: context, name: crashed_case[:test] )
      @file_wrapper.mkdir( File.dirname( log_path ) )
      @file_wrapper.write( log_path, "=== #{crashed_case[:test]} ===\n#{crash_result[:output]}\n", 'a' )

      unresolved.each do |test_case|
        group_results[:failed] += 1

        if !test_case.equal?( crashed_case )
          # An earlier case in this same parameterized group already crashed the
          # process -- this member never got a chance to run.
          group_results[:output] <<
            "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: " \
            "Test case not run -- an earlier case in this parameterized test group crashed"
          next
        end

        # Collect file_name and line in which crash occurred.
        # Match against the actual C symbol (`:symbol`), not the human-facing test name
        # (`:test`): a parameterized test case crashes inside a generated wrapper function
        # (`runner_args<N>_<test>`), not a function literally named `<test>(<args>)`.
        matched = crash_result[:output].match( /#{Regexp.escape(test_case[:symbol])}\s*\(\)\sat.+#{filename}:(\d+)\n/ )

        # If we found an error report line containing `test_case() at filename.c:###` in `gdb` output
        if matched
          # Line number
          line_number = matched[1]

          # Build terse signal label: "[SIGNAL] Description"
          signal_label = format_signal_label( crash_result[:output] )

          # Extract the offending source line (nil for assertion crashes or when unavailable)
          source_line = extract_source_line( crash_result[:output], test_case[:symbol], filename )

          # Unity's test executable output is line oriented.
          # Multi-line output is not possible (it looks like random `printf()` statements to the results parser).
          # "Encode" newlines in multiline string to be handled by the test results parser.
          crash_detail = source_line ? "#{NEWLINE_TOKEN}`#{source_line}`" : ''

          # Log path appears on its own encoded line so the results parser treats it separately
          group_results[:output] <<
            "#{filename}:#{line_number}:#{test_case[:test]}:FAIL: Test case crashed" \
            " >> #{signal_label}" \
            "#{crash_detail}" \
            "#{NEWLINE_TOKEN}(#{log_path})"

        # Try to extract a useful label even when no crash location frame was found.
        # A brief Windows assertion failure may report only the assertion text without frames.
        else
          label = format_signal_label( crash_result[:output] )

          if !label.empty?
            group_results[:output] <<
              "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed" \
              " >> #{label}" \
              "#{NEWLINE_TOKEN}(#{log_path})"
          else
            group_results[:output] <<
              "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: " \
              "Test case crashed (failed to extract `gdb` report)" \
              "#{NEWLINE_TOKEN}(#{log_path})"
          end
        end
      end
    end

    # Every member in this group resolved via a matched result line -- but gdb
    # normally exits successfully after handling a crash regardless of what happened
    # to the debuggee, so a clean-looking group here is a weaker signal than it is for
    # `do_simple`. Still checked for the same reason: a diagnostic retry's own real
    # status contradicting a fully clean set of matches is never trustworthy, whatever
    # tool produced it.
    if unresolved.empty? && @helper.test_crash?( filename, executable, crash_result )
      group_results = @RESULTS_COLLECTOR.new(
        passed: 0, ignored: 0, failed: group.size,
        output: group.map { |test_case|
          "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed" \
          " >> diagnostic retry's own exit status contradicts its reported clean result"
        }
      )
      unresolved = group # non-empty now, marks this group as having shown crash evidence
    end

    any_group_crashed ||= !unresolved.empty?

    test_case_results[:passed]  += group_results[:passed]
    test_case_results[:ignored] += group_results[:ignored]
    test_case_results[:failed]  += group_results[:failed]
    test_case_results[:output].concat( group_results[:output] )
  end

  # No retry group, across the entire file, ever showed real crash evidence, yet this
  # method only runs because the main run was already established as a crash. An
  # all-clean diagnostic must never be allowed to overrule that -- fall back to the
  # same whole-file crash-as-failure construction the :none backtrace setting uses.
  unless any_group_crashed
    @loginator.log(
      "Diagnostic retry for `#{File.basename(executable)}` could not reproduce the crash " \
      "already detected on the main run -- reporting the original crash rather than trusting " \
      "an all-clear diagnostic that contradicts it.",
      Verbosity::ERRORS, LogLabels::CRASH
    )
    return @generator_test_results.create_crash_failure( filename, shell_result, test_cases )
  end

  # Reset shell result exit code and output
  shell_result[:exit_code] = test_case_results[:failed]
  shell_result[:output] =
    @generator_test_results.regenerate_test_executable_stdout(
      total:   test_cases.size(),
      ignored: test_case_results[:ignored],
      failed:  test_case_results[:failed],
      output:  test_case_results[:output]
    )

  return shell_result
end

#do_simple(filename, executable, shell_result, test_cases, context:) ⇒ Object

Re-runs each test case (or, for a parameterized test, each group of parameterized cases -- see group_test_cases) individually to determine which one(s) crashed. For crash cases, captures any extra output from the test binary (e.g. assertion messages on stderr) and includes it in the failure report. Returns a modified shell_result with regenerated output.



223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
 
# File 'lib/ceedling/generators/generator_test_results_backtrace.rb', line 223

def do_simple(filename, executable, shell_result, test_cases, context:)
  # Clean stats tracker
  test_case_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )

  # Reset time
  shell_result[:time] = 0

  # True once some retry group has actually shown crash evidence of its own -- a
  # member with no result line, or (below) a group whose real status contradicts a
  # fully clean set of matches. If this stays false across every group, the whole
  # diagnostic never reproduced or attributed the crash the main run already
  # detected, and none of it can be trusted -- see the fallback after the loop.
  any_group_crashed = false

  # Iterate on test cases, one sub-process run per group (see `group_test_cases`)
  group_test_cases( test_cases ).each do |group|
    # Build the test fixture to run with our test case (or parameterized group) of interest
    command = @tool_executor.build_command_line(
      @configurator.tools_test_fixture_simple_backtrace, [],
      executable,
      unity_filter_arg( group )
    )
    # Things are gonna go boom, so ignore booms to get output
    command[:options][:boom] = false

    crash_result = @tool_executor.exec( command )

    # Sum execution time for each sub-process run
    # Note: Running tests separately increases total execution time
    shell_result[:time] += crash_result[:time].to_f()

    # Buffered separately from test_case_results and only merged in afterward, since
    # the status check below can still discard every match here in favor of a crash
    # attribution, once the whole group has been seen.
    group_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
    crashed = false # Has the actual crash in this group already been attributed?

    # Attribute each group member its own real result line, if Unity printed one
    group.each do |test_case|
      case crash_result[:output]
      # Success test case
      when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:PASS\s*$)/
        group_results[:passed]  += 1
        group_results[:output] << $1

      # Ignored test case
      when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:IGNORE\s*$)/
        group_results[:ignored] += 1
        group_results[:output] << $1

      when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:FAIL(:.+)?\s*$)/
        group_results[:failed]  += 1
        group_results[:output] << $1

      # No result line for this member -- either it crashed, or it never got to run
      # because an earlier member in this same group crashed.
      else
        group_results[:failed] += 1

        if crashed
          group_results[:output] <<
            "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: " \
            "Test case not run -- an earlier case in this parameterized test group crashed"
        else
          crashed = true
          # Collect any non-result, non-blank lines (e.g. assertion messages on stderr)
          extra = extract_simple_crash_output( crash_result[:output], filename )
          test_output = "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed"
          test_output += " >> #{extra.join(NEWLINE_TOKEN)}" unless extra.empty?
          group_results[:output] << test_output
        end
      end
    end

    # Every member in this group resolved via a matched result line -- but a diagnostic
    # retry running through its own, separately-configured tool (e.g. without the main
    # :test_fixture's sanitizer options) can complete cleanly and print a legitimate-
    # looking PASS even when it isn't a trustworthy stand-in for what actually happened.
    # The same real-status check the main run itself relies on applies here too.
    if !crashed && @helper.test_crash?( filename, executable, crash_result )
      crashed = true
      group_results = @RESULTS_COLLECTOR.new(
        passed: 0, ignored: 0, failed: group.size,
        output: group.map { |test_case|
          "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed" \
          " >> diagnostic retry's own exit status contradicts its reported clean result"
        }
      )
    end

    any_group_crashed ||= crashed

    test_case_results[:passed]  += group_results[:passed]
    test_case_results[:ignored] += group_results[:ignored]
    test_case_results[:failed]  += group_results[:failed]
    test_case_results[:output].concat( group_results[:output] )
  end

  # No retry group, across the entire file, ever showed real crash evidence, yet this
  # method only runs because the main run was already established as a crash. An
  # all-clean diagnostic must never be allowed to overrule that -- fall back to the
  # same whole-file crash-as-failure construction the :none backtrace setting uses.
  unless any_group_crashed
    @loginator.log(
      "Diagnostic retry for `#{File.basename(executable)}` could not reproduce the crash " \
      "already detected on the main run -- reporting the original crash rather than trusting " \
      "an all-clear diagnostic that contradicts it.",
      Verbosity::ERRORS, LogLabels::CRASH
    )
    return @generator_test_results.create_crash_failure( filename, shell_result, test_cases )
  end

  # Reset shell result exit code and output
  shell_result[:exit_code] = test_case_results[:failed]
  shell_result[:output] =
    @generator_test_results.regenerate_test_executable_stdout(
      total:   test_cases.size(),
      ignored: test_case_results[:ignored],
      failed:  test_case_results[:failed],
      output:  test_case_results[:output]
    )

  return shell_result
end

#setup ⇒ Object 



13
14
15
16
17
 
# File 'lib/ceedling/generators/generator_test_results_backtrace.rb', line 13

def setup()
  @RESULTS_COLLECTOR = Struct.new( :passed, :failed, :ignored, :output, keyword_init:true )
  # Alias, matching Generator's own convention for the same dependency.
  @helper = @generator_helper
end