Module: MilkTea::CLI::CommandTest

Included in:
MilkTea::CLI
Defined in:
lib/milk_tea/tooling/cli/commands/test.rb

Defined Under Namespace

Classes: TestResult

Constant Summary collapse

TEST_RUN_TIMEOUT_SECONDS = 
30
TEST_RUN_MEMORY_LIMIT_BYTES = 
1024 * 1024 * 1024

Instance Method Summary collapse

Instance Method Details

#classify_death_test(binary_path) ⇒ Object 



451
452
453
454
455
456
457
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 451

def classify_death_test(binary_path)
  _output, status, timed_out = spawn_sandboxed(binary_path)
  return :timed_out if timed_out
  return :returned if status&.exited? && status.exitstatus&.zero?

  :aborted
end

#classify_test_file(file) ⇒ Object 



346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 346

def classify_test_file(file)
  source = File.read(file)
  if compile_fail_fixture?(source)
    return nil if @test_filter && !File.basename(file, ".mt").include?(@test_filter)

    return :compile_fail
  end

  ast = begin
    MilkTea::Parser.parse(source, path: file)
  rescue ParseError
    return nil
  end
  has_match = ast.declarations.any? do |decl|
    decl.is_a?(AST::FunctionDef) && test_attribute?(decl) && matches_filter?(decl.name)
  end
  has_match ? :test : nil
end

#compile_fail_diagnostics(path) ⇒ Object 



260
261
262
263
264
265
266
267
268
269
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 260

def compile_fail_diagnostics(path)
  errors, = check_single_reporting_all(path, locked: false)
  errors
    .select { |diagnostic| !diagnostic.respond_to?(:severity) || diagnostic.severity == :error }
    .map { |diagnostic| ErrorFormatter.format(diagnostic, color: false) }
rescue StandardError => e
  raise unless handled_cli_error?(e)

  [ErrorFormatter.format(e, color: false)]
end

#compile_fail_fixture?(source) ⇒ Boolean

Returns:

  • (Boolean) 


365
366
367
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 365

def compile_fail_fixture?(source)
  source.match?(/^\s*#\s*expect-error:/)
end

#death_test_runner_main(test_name) ⇒ Object 



459
460
461
462
463
464
465
466
467
468
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 459

def death_test_runner_main(test_name)
  [
    "function main() -> int:",
    "    match #{test_name}():",
    "        Result.success:",
    "            return 0",
    "        Result.failure:",
    "            return 0",
  ].join("\n") + "\n"
end

#discover_test_files(directory) ⇒ Object 



337
338
339
340
341
342
343
344
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 337

def discover_test_files(directory)
  Dir.glob(File.join(directory, "**", "*.mt")).sort.filter_map do |file|
    next if File.basename(file).start_with?("__mt_test_runner_")

    kind = classify_test_file(file)
    kind && [file, kind]
  end
end

#dispatch_test_run(path, options:, locked:) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 42

def dispatch_test_run(path, options:, locked:)
  if File.directory?(path)
    run_test_directory(path, options:, locked:)
  elsif File.file?(path)
    if compile_fail_fixture?(File.read(path))
      run_compile_fail_test(path) ? 0 : 1
    else
      run_test_file(path, options:, locked:)
    end
  else
    @err.puts("mtc test: not a file or directory: #{path}")
    1
  end
end

#emit_junit(results, out) ⇒ Object 



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 171

def emit_junit(results, out)
  failures = results.count { |result| result.status == :fail }
  skipped = results.count { |result| result.status == :skip }
  out.puts(%(<?xml version="1.0" encoding="UTF-8"?>))
  out.puts(%(<testsuites tests="#{results.length}" failures="#{failures}" skipped="#{skipped}">))
  out.puts(%(  <testsuite name="mtc test" tests="#{results.length}" failures="#{failures}" skipped="#{skipped}">))
  results.each do |result|
    name = xml_escape(result.name)
    case result.status
    when :pass
      out.puts(%(    <testcase name="#{name}"/>))
    when :skip
      out.puts(%(    <testcase name="#{name}"><skipped/></testcase>))
    when :fail
      out.puts(%(    <testcase name="#{name}"><failure message="#{xml_escape(result.detail || 'failed')}"/></testcase>))
    end
  end
  out.puts("  </testsuite>")
  out.puts("</testsuites>")
end

#emit_machine_results(results, format, out) ⇒ Object 



143
144
145
146
147
148
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 143

def emit_machine_results(results, format, out)
  case format
  when :tap then emit_tap(results, out)
  when :junit then emit_junit(results, out)
  end
end

#emit_tap(results, out) ⇒ Object 



150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 150

def emit_tap(results, out)
  out.puts("TAP version 13")
  out.puts("1..#{results.length}")
  results.each_with_index do |result, index|
    number = index + 1
    case result.status
    when :pass
      out.puts("ok #{number} - #{result.name}")
    when :skip
      out.puts("ok #{number} - #{result.name} # SKIP#{result.detail ? " #{result.detail}" : ''}")
    when :fail
      out.puts("not ok #{number} - #{result.name}")
      next unless result.detail

      out.puts("  ---")
      out.puts("  message: #{result.detail}")
      out.puts("  ...")
    end
  end
end

#expect_fatal_attribute?(decl) ⇒ Boolean

Returns:

  • (Boolean) 


426
427
428
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 426

def expect_fatal_attribute?(decl)
  decl.attributes.any? { |attribute| attribute.name.parts == ["expect_fatal"] }
end

#extract_expect_error_directives(source) ⇒ Object 



253
254
255
256
257
258
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 253

def extract_expect_error_directives(source)
  source.each_line.filter_map do |line|
    match = line.match(/^\s*#\s*expect-error:\s*(.+?)\s*$/)
    match && match[1]
  end
end

#extract_test_limit_flags! ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 57

def extract_test_limit_flags!
  timeout_seconds = TEST_RUN_TIMEOUT_SECONDS
  memory_bytes = TEST_RUN_MEMORY_LIMIT_BYTES
  jobs = 1
  sanitize = false
  filter = nil
  format = :human
  remaining = []
  until @argv.empty?
    arg = @argv.shift
    case arg
    when "--timeout"
      value = @argv.shift
      seconds = value && Integer(value, exception: false)
      unless seconds&.positive?
        @err.puts("--timeout requires a positive integer (seconds)")
        return nil
      end
      timeout_seconds = seconds
    when "--mem"
      value = @argv.shift
      megabytes = value && Integer(value, exception: false)
      unless megabytes&.positive?
        @err.puts("--mem requires a positive integer (megabytes)")
        return nil
      end
      memory_bytes = megabytes * 1024 * 1024
    when "--jobs"
      value = @argv.shift
      count = value && Integer(value, exception: false)
      unless count&.positive?
        @err.puts("--jobs requires a positive integer")
        return nil
      end
      jobs = count
    when "--sanitize"
      sanitize = true
    when "-n", "--name"
      value = @argv.shift
      unless value
        @err.puts("-n requires a name substring")
        return nil
      end
      filter = value
    when "--format"
      value = @argv.shift
      unless %w[human tap junit].include?(value)
        @err.puts("--format must be human, tap, or junit")
        return nil
      end
      format = value.to_sym
    when "--"
      remaining << arg
      remaining.concat(@argv)
      @argv.clear
    else
      remaining << arg
    end
  end
  @argv.replace(remaining)
  [timeout_seconds, memory_bytes, jobs, sanitize, filter, format]
end

#matches_filter?(name) ⇒ Boolean

Returns:

  • (Boolean) 


373
374
375
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 373

def matches_filter?(name)
  @test_filter.nil? || name.include?(@test_filter)
end

#parse_test_output(text) ⇒ Object 



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 120

def parse_test_output(text)
  results = []
  current_file = nil
  text.each_line do |raw|
    line = raw.chomp
    case line
    when /\A# (.+)\z/
      current_file = ::Regexp.last_match(1)
    when /\Aok   - (.+)\z/
      results << TestResult.new(name: ::Regexp.last_match(1), status: :pass, detail: nil)
    when /\Askip - (.+?)(?:: (.*))?\z/
      results << TestResult.new(name: ::Regexp.last_match(1), status: :skip, detail: ::Regexp.last_match(2))
    when /\AFAIL - (.+?)(?:: (.*))?\z/
      results << TestResult.new(name: ::Regexp.last_match(1), status: :fail, detail: ::Regexp.last_match(2))
    when /\AFAILED - (.+?) \(build error\)\z/
      results << TestResult.new(name: "#{::Regexp.last_match(1)} (build error)", status: :fail, detail: "build error")
    when /\Atest run (?:timed out|crashed)/, /\ASUMMARY: \w+Sanitizer/
      results << TestResult.new(name: "#{current_file || 'test'} (#{line})", status: :fail, detail: line)
    end
  end
  results
end

#run_classified_file(file, kind, options:, locked:) ⇒ Object 



222
223
224
225
226
227
228
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 222

def run_classified_file(file, kind, options:, locked:)
  if kind == :compile_fail
    run_compile_fail_test(file) ? 0 : 1
  else
    run_test_file_guarded(file, options:, locked:)
  end
end

#run_compile_fail_test(path) ⇒ Object 



230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 230

def run_compile_fail_test(path)
  source = File.read(path)
  expectations = extract_expect_error_directives(source)
  messages = compile_fail_diagnostics(path)

  if messages.empty?
    @out.puts("FAIL - #{path} (compile-fail): expected a compile error, but it compiled cleanly")
    @out.flush if @out.respond_to?(:flush)
    return false
  end

  unmatched = expectations.find { |expected| messages.none? { |message| message.include?(expected) } }
  if unmatched
    @out.puts("FAIL - #{path} (compile-fail): no diagnostic matched #{unmatched.inspect}")
    @out.flush if @out.respond_to?(:flush)
    return false
  end

  @out.puts("ok   - #{path} (compile-fail)")
  @out.flush if @out.respond_to?(:flush)
  true
end

#run_death_test(source_path, source, test_name, options:, locked:) ⇒ Object 



430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 430

def run_death_test(source_path, source, test_name, options:, locked:)
  runner_source = source.dup
  runner_source << "\n\n" << death_test_runner_main(test_name)
  classification = with_synthesized_binary(source_path, runner_source, options:, locked:) do |binary_path|
    classify_death_test(binary_path)
  end

  passed = classification == :aborted
  line =
    if passed
      "ok   - #{test_name} (expect_fatal)"
    elsif classification == :timed_out
      "FAIL - #{test_name} (expect_fatal): timed out"
    else
      "FAIL - #{test_name} (expect_fatal): expected a fatal abort, but the test returned"
    end
  @out.puts(line)
  @out.flush if @out.respond_to?(:flush)
  passed
end

#run_synthesized_tests(source_path, runner_source, options:, locked:) ⇒ Object 



480
481
482
483
484
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 480

def run_synthesized_tests(source_path, runner_source, options:, locked:)
  with_synthesized_binary(source_path, runner_source, options:, locked:) do |binary_path|
    run_test_binary(binary_path)
  end
end

#run_test_binary(binary_path) ⇒ Object 



511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 511

def run_test_binary(binary_path)
  output, status, timed_out = spawn_sandboxed(binary_path)
  @out.write(output)
  @out.flush if @out.respond_to?(:flush)

  if timed_out
    @err.puts("test run timed out after #{@test_timeout_seconds || TEST_RUN_TIMEOUT_SECONDS}s")
    return 1
  end
  if status&.signaled?
    @err.puts("test run crashed (signal #{status.termsig})")
    return 1
  end

  status&.exitstatus || 1
end

#run_test_directory(directory, options:, locked:) ⇒ Object 



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 196

def run_test_directory(directory, options:, locked:)
  test_files = discover_test_files(directory)
  if test_files.empty?
    if @test_filter
      @out.puts("no tests matched -n '#{@test_filter}' under #{directory}")
    else
      @out.puts("no @[test] functions or # expect-error: fixtures found under #{directory}")
    end
    return 0
  end

  jobs = @test_jobs || 1
  return run_test_files_parallel(test_files, jobs:, options:, locked:) if jobs > 1 && Process.respond_to?(:fork)

  failed = 0
  test_files.each do |file, kind|
    @out.puts("# #{file}")
    @out.flush if @out.respond_to?(:flush)
    failed += 1 unless run_classified_file(file, kind, options:, locked:).zero?
  end

  @out.puts("")
  @out.puts("#{test_files.length} test file(s), #{failed} failed")
  failed.zero? ? 0 : 1
end

#run_test_file(path, options:, locked:) ⇒ Object 



377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 377

def run_test_file(path, options:, locked:)
  source = File.read(path)
  ast = MilkTea::Parser.parse(source, path:)

  if ast.declarations.any? { |decl| decl.is_a?(AST::FunctionDef) && decl.name == "main" }
    @err.puts("a test file must not define 'main': #{path}")
    return 1
  end

  tests = ast.declarations.select { |decl| decl.is_a?(AST::FunctionDef) && test_attribute?(decl) }

  if tests.empty?
    @out.puts("no @[test] functions found in #{path}")
    return 0
  end

  tests = tests.select { |test| matches_filter?(test.name) }
  return 0 if tests.empty?

  invalid = tests.find { |test| !test.params.empty? }
  if invalid
    @err.puts("@[test] function '#{invalid.name}' must take no parameters")
    return 1
  end

  testing_import = ast.imports.find { |import| import.path.parts == %w[std testing] }
  unless testing_import
    @err.puts("a test file must import std.testing: #{path}")
    return 1
  end
  testing_alias = testing_import.alias_name || testing_import.path.parts.last

  death_tests, normal_tests = tests.partition { |test| expect_fatal_attribute?(test) }

  exit_code = 0

  unless normal_tests.empty?
    runner_source = source.dup
    runner_source << "\n\n" << test_runner_main(testing_alias, normal_tests.map(&:name))
    exit_code = run_synthesized_tests(path, runner_source, options:, locked:)
  end

  death_tests.each do |death_test|
    exit_code = 1 unless run_death_test(path, source, death_test.name, options:, locked:)
  end

  exit_code
end

#run_test_file_guarded(file, options:, locked:) ⇒ Object 



271
272
273
274
275
276
277
278
279
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 271

def run_test_file_guarded(file, options:, locked:)
  run_test_file(file, options:, locked:)
rescue StandardError => e
  raise unless handled_cli_error?(e)

  @err.puts("FAILED - #{file} (build error)")
  @err.puts(ErrorFormatter.format(e, color: error_color?(@err)))
  1
end

#run_test_files_parallel(test_files, jobs:, options:, locked:) ⇒ Object 



281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 281

def run_test_files_parallel(test_files, jobs:, options:, locked:)
  results = Array.new(test_files.length)
  result_paths = {}
  active = {}
  cursor = 0

  while cursor < test_files.length || !active.empty?
    while active.size < jobs && cursor < test_files.length
      index = cursor
      cursor += 1
      result_path = File.join(Dir.tmpdir, "mttest_result_#{Process.pid}_#{index}")
      result_paths[index] = result_path
      pid = fork do
        captured = StringIO.new
        @out = captured
        @err = captured
        file, kind = test_files[index]
        code = run_classified_file(file, kind, options:, locked:)
        File.binwrite(result_path, [code].pack("N") + captured.string)
        exit!(0)
      end
      active[pid] = index
    end

    finished_pid, = Process.wait2
    index = active.delete(finished_pid)
    next unless index

    path = result_paths[index]
    data = begin
      File.binread(path)
    rescue StandardError
      (+"").b
    end
    File.delete(path) if File.exist?(path)
    results[index] =
      if data.bytesize >= 4
        [data[0, 4].unpack1("N"), data.byteslice(4..).force_encoding(Encoding::UTF_8)]
      else
        [1, +""]
      end
  end

  failed = 0
  test_files.each_with_index do |(file, _kind), index|
    code, output = results[index]
    @out.puts("# #{file}")
    @out.write(output.to_s)
    failed += 1 unless code&.zero?
  end
  @out.flush if @out.respond_to?(:flush)
  @out.puts("")
  @out.puts("#{test_files.length} test file(s), #{failed} failed")
  failed.zero? ? 0 : 1
end

#spawn_sandboxed(binary_path) ⇒ Object 



528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 528

def spawn_sandboxed(binary_path)
  timeout_seconds = @test_timeout_seconds || TEST_RUN_TIMEOUT_SECONDS
  memory_bytes = @test_memory_bytes || TEST_RUN_MEMORY_LIMIT_BYTES
  reader, writer = IO.pipe
  spawn_options = { out: writer, err: writer, pgroup: true }
  spawn_options[:rlimit_as] = memory_bytes unless @test_sanitize
  pid = Process.spawn(binary_path, **spawn_options)
  writer.close

  status = nil
  timed_out = false
  begin
    Timeout.timeout(timeout_seconds) { _, status = Process.wait2(pid) }
  rescue Timeout::Error
    timed_out = true
    begin
      Process.kill("-KILL", Process.getpgid(pid))
      Process.wait(pid)
    rescue StandardError
      nil
    end
  end

  output = reader.read
  reader.close
  [output, status, timed_out]
end

#test_attribute?(decl) ⇒ Boolean

Returns:

  • (Boolean) 


369
370
371
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 369

def test_attribute?(decl)
  decl.attributes.any? { |attribute| attribute.name.parts == ["test"] }
end

#test_command ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 11

def test_command
  limits = extract_test_limit_flags!
  return 1 unless limits

  @test_timeout_seconds, @test_memory_bytes, @test_jobs, @test_sanitize, @test_filter, @test_format = limits

  path, options = extract_path_and_options
  return 1 unless path

  frozen = options.delete(:frozen)
  ensure_current_lockfile!(path) if frozen
  locked = options.delete(:locked)

  return dispatch_test_run(path, options:, locked:) if @test_format == :human

  buffer = StringIO.new
  real_out = @out
  real_err = @err
  @out = buffer
  @err = buffer
  begin
    exit_code = dispatch_test_run(path, options:, locked:)
  ensure
    @out = real_out
    @err = real_err
  end

  emit_machine_results(parse_test_output(buffer.string), @test_format, real_out)
  exit_code
end

#test_runner_main(testing_alias, test_names) ⇒ Object 



470
471
472
473
474
475
476
477
478
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 470

def test_runner_main(testing_alias, test_names)
  lines = ["function main() -> int:"]
  lines << "    var __mt_test_stats = #{testing_alias}.Stats.create()"
  test_names.each do |name|
    lines << "    __mt_test_stats = #{testing_alias}.record(__mt_test_stats, #{name.inspect}, #{name}())"
  end
  lines << "    return #{testing_alias}.summarize(__mt_test_stats)"
  lines.join("\n") + "\n"
end

#with_synthesized_binary(source_path, runner_source, options:, locked:) ⇒ Object 



486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 486

def with_synthesized_binary(source_path, runner_source, options:, locked:)
  directory = File.dirname(File.expand_path(source_path))
  runner_path = File.join(directory, "__mt_test_runner_#{Process.pid}.mt")
  binary_path = File.join(Dir.tmpdir, "__mt_test_runner_#{Process.pid}")

  File.write(runner_path, runner_source)
  begin
    build_opts = options.except(:timings, :output_path, :bundle, :archive)
    build_opts[:debug_guards] = false
    Build.build(
      runner_path,
      output_path: binary_path,
      module_roots: module_roots_for(source_path, locked:),
      package_graph: package_graph_for(source_path, locked:),
      frontend: @build_frontend,
      sanitize: @test_sanitize,
      **build_opts,
    )
    yield binary_path
  ensure
    File.delete(runner_path) if File.exist?(runner_path)
    File.delete(binary_path) if File.exist?(binary_path)
  end
end

#xml_escape(value) ⇒ Object 



192
193
194
 
# File 'lib/milk_tea/tooling/cli/commands/test.rb', line 192

def xml_escape(value)
  value.to_s.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;").gsub('"', "&quot;")
end