Class: Datadog::CI::TestSuite

Inherits:
ConcurrentSpan show all
Defined in:
lib/datadog/ci/test_suite.rb

Overview

Represents a single test suite.

Read here on what test suite means: https://docs.datadoghq.com/continuous_integration/explorer/?tab=testruns#suite

This object can be shared between multiple threads.

Instance Attribute Summary

Attributes inherited from Span

#tracer_span

Instance Method Summary collapse

Methods inherited from ConcurrentSpan

#get_tag, #set_metric, #set_tag, #set_tags, #synchronize

Methods inherited from Span

#base_commit_sha, #clear_tag, #failed!, #failed?, #get_metric, #get_tag, #git_branch, #git_commit_message, #git_commit_sha, #git_repository_url, #git_tag, #id, #name, #original_git_commit_message, #original_git_commit_sha, #os_architecture, #os_platform, #os_version, #passed!, #passed?, #runtime_name, #runtime_version, #service, #set_default_tags, #set_environment_runtime_tags, #set_metric, #set_tag, #set_tags, #skipped!, #skipped?, #source_file, #status, #to_s, #trace_id, #type, #undefined?

Constructor Details

#initialize(tracer_span) ⇒ TestSuite

Returns a new instance of TestSuite.



16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/datadog/ci/test_suite.rb', line 16

def initialize(tracer_span)
  super

  @custom_impacted_files = []

  # counts how many times every test in this suite was executed with each status:
  #   { "MySuite.mytest.a:1" => { "pass" => 3, "fail" => 2 } }
  @execution_stats_per_test = {}

  # tracks final status for each test (the status that is reported after all retries):
  #   { "MySuite.mytest.a:1" => "pass" }
  @final_statuses_per_test = {}
end

Instance Method Details

#add_impacted_files(file_paths) ⇒ void

This method returns an undefined value.

Adds files that Test Impact Analysis should consider capable of impacting every test in this suite.

In test-level skipping mode, custom impacted files must be added before the first test in this suite starts. In suite-level skipping mode, they may be added until the suite finishes.

This is useful for files shared by every test that Datadog's native Ruby coverage cannot observe, such as JavaScript test setup. Calls are incremental: every call adds files for the remainder of the suite.

Paths must resolve inside the Git repository. Relative paths must be relative to the repository root. Absolute paths are also accepted.

If paths are relative to the current working directory, convert them to absolute paths with File.expand_path before calling this method. Submitted paths must be lexically normalized without redundant . or .. components. Resolution does not access the filesystem. Paths outside the repository are ignored when the coverage event is serialized.

Examples:

Register JavaScript setup shared by an RSpec suite

before(:context) do
  Datadog::CI.active_test_suite&.add_impacted_files(
    ["app/frontend/test_setup.js"]
  )
end

Parameters:

  • file_paths (Array<String>) —

    custom paths that can impact this suite

Raises:

  • (RuntimeError) —

    if test-level skipping is active and a test in this suite has already started

  • (ArgumentError) —

    if file_paths is not an Array



63
64
65
66
67
68
69
70
71
 
# File 'lib/datadog/ci/test_suite.rb', line 63

def add_impacted_files(file_paths)
  raise ArgumentError, "file_paths must be an Array" unless file_paths.is_a?(Array)

  synchronize do
    ensure_custom_impacted_files_mutable!
    @custom_impacted_files.concat(file_paths)
  end
  nil
end

#all_executions_failed?(test_id) ⇒ Boolean

Returns:

  • (Boolean) 


133
134
135
136
137
138
 
# File 'lib/datadog/ci/test_suite.rb', line 133

def all_executions_failed?(test_id)
  synchronize do
    stats = @execution_stats_per_test[test_id]
    stats && stats[Ext::Test::Status::FAIL] > 0 && stats[Ext::Test::Status::PASS] == 0
  end
end

#all_executions_passed?(test_id) ⇒ Boolean

Returns:

  • (Boolean) 


141
142
143
144
145
146
 
# File 'lib/datadog/ci/test_suite.rb', line 141

def all_executions_passed?(test_id)
  synchronize do
    stats = @execution_stats_per_test[test_id]
    stats && stats[Ext::Test::Status::PASS] > 0 && stats[Ext::Test::Status::FAIL] == 0
  end
end

#any_passed? ⇒ Boolean

Returns:

  • (Boolean) 


116
117
118
119
120
121
122
 
# File 'lib/datadog/ci/test_suite.rb', line 116

def any_passed?
  synchronize do
    @execution_stats_per_test.any? do |_, stats|
      stats[Ext::Test::Status::PASS] > 0
    end
  end
end

#any_test_retry_passed?(test_id) ⇒ Boolean

Returns:

  • (Boolean) 


125
126
127
128
129
130
 
# File 'lib/datadog/ci/test_suite.rb', line 125

def any_test_retry_passed?(test_id)
  synchronize do
    stats = @execution_stats_per_test[test_id]
    stats && stats[Ext::Test::Status::PASS] > 0
  end
end

#datadog_skip_reason ⇒ Object 



174
175
176
 
# File 'lib/datadog/ci/test_suite.rb', line 174

def datadog_skip_reason
  get_tag(Ext::Test::TAG_SKIP_REASON)
end

#expected_test_done!(test_name) ⇒ Object 



165
166
167
168
169
170
171
 
# File 'lib/datadog/ci/test_suite.rb', line 165

def expected_test_done!(test_name)
  synchronize do
    @expected_tests_set.delete(test_name)

    finish if @expected_tests_set.empty?
  end
end

#finish ⇒ void

This method returns an undefined value.

Finishes this test suite.



89
90
91
92
93
94
95
96
97
98
 
# File 'lib/datadog/ci/test_suite.rb', line 89

def finish
  synchronize do
    # we try to derive test suite status from execution stats if no status was set explicitly
    set_status_from_stats! if undefined?

    test_tracing.deactivate_test_suite(name)

    super
  end
end

#itr_unskippable? ⇒ Boolean

Returns:

  • (Boolean) 


189
190
191
 
# File 'lib/datadog/ci/test_suite.rb', line 189

def itr_unskippable?
  get_tag(Ext::Test::TAG_ITR_UNSKIPPABLE) == "true"
end

#lock_custom_impacted_files ⇒ Array<String>

Locks suite-level custom impacted files against further changes and returns them to Test Impact Analysis.

Returns:

  • (Array<String>) 


78
79
80
81
82
83
84
85
 
# File 'lib/datadog/ci/test_suite.rb', line 78

def lock_custom_impacted_files
  synchronize do
    unless @custom_impacted_files.frozen?
      @custom_impacted_files = @custom_impacted_files.uniq.freeze
    end
    @custom_impacted_files
  end
end

#record_test_final_status(test_id, final_status) ⇒ Object 



109
110
111
112
113
 
# File 'lib/datadog/ci/test_suite.rb', line 109

def record_test_final_status(test_id, final_status)
  synchronize do
    @final_statuses_per_test[test_id] = final_status
  end
end

#record_test_result(test_id, datadog_test_status) ⇒ Object 



101
102
103
104
105
106
 
# File 'lib/datadog/ci/test_suite.rb', line 101

def record_test_result(test_id, datadog_test_status)
  synchronize do
    @execution_stats_per_test[test_id] ||= Hash.new(0)
    @execution_stats_per_test[test_id][datadog_test_status] += 1
  end
end

#set_expected_tests!(expected_tests) ⇒ Object 



156
157
158
159
160
161
162
 
# File 'lib/datadog/ci/test_suite.rb', line 156

def set_expected_tests!(expected_tests)
  synchronize do
    return if @expected_tests_set

    @expected_tests_set = Set.new(expected_tests)
  end
end

#should_skip? ⇒ Boolean

Returns:

  • (Boolean) 


179
180
181
 
# File 'lib/datadog/ci/test_suite.rb', line 179

def should_skip?
  skipped_by_test_impact_analysis?
end

#skipped_by_test_impact_analysis? ⇒ Boolean

Returns:

  • (Boolean) 


184
185
186
 
# File 'lib/datadog/ci/test_suite.rb', line 184

def skipped_by_test_impact_analysis?
  get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
end

#test_executed?(test_id) ⇒ Boolean

Returns:

  • (Boolean) 


149
150
151
152
153
 
# File 'lib/datadog/ci/test_suite.rb', line 149

def test_executed?(test_id)
  synchronize do
    @execution_stats_per_test.key?(test_id)
  end
end