Module: Ibex::CLIFuzzRegressions

Included in:
CLIFuzz
Defined in:
lib/ibex/cli/fuzz_regressions.rb,
sig/ibex/cli/fuzz_regressions.rbs

Overview

Automatic bounded minimization and persistence for fuzz differences.

Instance Method Summary collapse

Instance Method Details

#fuzz_regression_document(mismatch, result, grammar_path, external) ⇒ Hash[Symbol, Object?]

RBS:

  • (fuzz_mismatch mismatch, DeltaReducer::Result result, String grammar_path, fuzz_external_description? external) -> Hash[Symbol, Object?]

Parameters:

  • mismatch (fuzz_mismatch)
  • result (DeltaReducer::Result)
  • grammar_path (String)
  • external (fuzz_external_description, nil)

Returns:

  • (Hash[Symbol, Object?]) 


128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
 
# File 'lib/ibex/cli/fuzz_regressions.rb', line 128

def fuzz_regression_document(mismatch, result, grammar_path, external)
  document = {
    ibex_report: "fuzz-regression", schema_version: 1, grammar: grammar_path,
    seed: @options.fetch(:fuzz_seed, 0), kind: mismatch[:kind],
    sentence: mismatch[:sentence],
    algorithms: mismatch[:outcomes].keys.reject { |name| name == :external }.map(&:to_s),
    original_tokens: mismatch[:tokens], minimized_tokens: result.items,
    bounds: mismatch[:bounds],
    reduction: {
      original_size: result.original_size, minimized_size: result.items.length,
      trials: result.trials, complete: result.complete
    }
  } #: Hash[Symbol, Object?]
  document[:external] = external if external
  document
end

#minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external) ⇒ Hash[Symbol, untyped]

RBS:

  • (Fuzz fuzzer, Fuzz::Mismatch mismatch, String grammar_path, fuzz_external_target? external) -> Hash[Symbol, untyped]

Parameters:

  • fuzzer (Fuzz)
  • mismatch (Fuzz::Mismatch)
  • grammar_path (String)
  • external (fuzz_external_target, nil)

Returns:

  • (Hash[Symbol, untyped]) 


85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/ibex/cli/fuzz_regressions.rb', line 85

def minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external)
  result = fuzzer.minimize(
    mismatch, max_trials: @options.fetch(:fuzz_max_reduction_trials, 1_000)
  )
  details = mismatch.details.dup #: Hash[Symbol, untyped]
  details[:minimized_tokens] = result.items
  details[:reduction] = {
    original_size: result.original_size, minimized_size: result.items.length,
    trials: result.trials, complete: result.complete
  }
  regression = persist_fuzz_regression(
    mismatch.details, result, grammar_path, external&.[](1)
  )
  details[:regression] = regression if regression
  details
end

#persist_fuzz_regression(mismatch, result, grammar_path, external) ⇒ Hash[Symbol, String]?

RBS:

  • (fuzz_mismatch mismatch, DeltaReducer::Result result, String grammar_path, fuzz_external_description? external) -> Hash[Symbol, String]?

Parameters:

  • mismatch (fuzz_mismatch)
  • result (DeltaReducer::Result)
  • grammar_path (String)
  • external (fuzz_external_description, nil)

Returns:

  • (Hash[Symbol, String], nil) 


104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/ibex/cli/fuzz_regressions.rb', line 104

def persist_fuzz_regression(mismatch, result, grammar_path, external)
  return if @options.fetch(:fuzz_save_regression, true) == false

  directory = @options.fetch(:fuzz_regression_dir, "test/fuzz/regressions")
  FileUtils.mkdir_p(directory)
  document = fuzz_regression_document(mismatch, result, grammar_path, external)
  source = "#{JSON.pretty_generate(document)}\n"
  digest = Digest::SHA256.hexdigest(source)
  seed = @options.fetch(:fuzz_seed, 0)
  path = File.join(directory, "fuzz-seed-#{seed}-#{digest.slice(0, 16)}.json")
  validate_fuzz_regression_target!(path)
  atomic_write_ir(path, source)
  { path: path, sha256: digest }
end

#validate_fuzz_regression_target!(path) ⇒ void

This method returns an undefined value.

RBS:

  • (String path) -> void

Parameters:

  • path (String) 


120
121
122
123
124
 
# File 'lib/ibex/cli/fuzz_regressions.rb', line 120

def validate_fuzz_regression_target!(path)
  return unless File.symlink?(path) || (File.exist?(path) && File.stat(path).nlink > 1)

  raise Ibex::Error, "(fuzz):1:1: regression output refuses symlinks and files with multiple hard links"
end

#write_fuzz_budget_report(error, external, phase:) ⇒ Integer

RBS:

  • (Fuzz::BudgetExceeded error, fuzz_external_target? external, phase: String) -> Integer

Parameters:

Returns:

  • (Integer) 


73
74
75
76
77
78
79
80
81
 
# File 'lib/ibex/cli/fuzz_regressions.rb', line 73

def write_fuzz_budget_report(error, external, phase:)
  report = {
    ibex_report: "fuzz", schema_version: 1, result: "budget_exhausted",
    budget: error.details.merge(phase: phase)
  } #: Hash[Symbol, untyped]
  report[:external] = external[1] if external
  write_fuzz_report(report)
  2
end

#write_fuzz_report(report) ⇒ void

This method returns an undefined value.

RBS:

  • (Hash[Symbol, untyped] report) -> void

Parameters:

  • report (Hash[Symbol, untyped]) 


44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/ibex/cli/fuzz_regressions.rb', line 44

def write_fuzz_report(report)
  if @options.fetch(:fuzz_format, "json") == "json"
    @stdout.puts JSON.pretty_generate(report)
    return
  end

  @stdout.puts("result=#{report.fetch(:result)}")
  case report.fetch(:result)
  when "no_difference_within_bounds"
    @stdout.puts("seed=#{report.fetch(:seed)} bounds=#{report.fetch(:bounds).inspect}")
    @stdout.puts("no difference found within the declared bounds; this is not a proof of equivalence")
  when "difference"
    mismatch = report.fetch(:mismatch) #: Hash[Symbol, untyped]
    @stdout.puts("kind=#{mismatch.fetch(:kind)} tokens=#{mismatch.fetch(:tokens).inspect}")
    @stdout.puts("minimized=#{mismatch.fetch(:minimized_tokens).inspect} " \
                 "reduction_complete=#{mismatch.dig(:reduction, :complete)}")
    @stdout.puts("regression=#{mismatch.dig(:regression, :path)}") if mismatch[:regression]
  when "budget_exhausted"
    @stdout.puts("search incomplete: #{report.dig(:budget, :message)}")
  end
  external = report[:external] #: fuzz_external_description?
  return unless external

  @stdout.puts("target_runtime=#{external.fetch(:target_runtime)}")
  @stdout.puts("target_command=#{external.fetch(:command)}")
end

#write_minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external) ⇒ Integer

RBS:

  • (Fuzz fuzzer, Fuzz::Mismatch mismatch, String grammar_path, fuzz_external_target? external) -> Integer

Parameters:

  • fuzzer (Fuzz)
  • mismatch (Fuzz::Mismatch)
  • grammar_path (String)
  • external (fuzz_external_target, nil)

Returns:

  • (Integer) 


33
34
35
36
37
38
39
40
41
 
# File 'lib/ibex/cli/fuzz_regressions.rb', line 33

def write_minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external)
  details = minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external)
  report = {
    ibex_report: "fuzz", schema_version: 1, result: "difference", mismatch: details
  } #: Hash[Symbol, untyped]
  report[:external] = external[1] if external
  write_fuzz_report(report)
  1
end