Class: Arachni::BrowserCluster

Inherits:
Object
  • Object
show all
Includes:
UI::Output, Utilities
Defined in:
lib/arachni/browser_cluster.rb,
lib/arachni/browser_cluster/job.rb,
lib/arachni/browser_cluster/worker.rb,
lib/arachni/browser_cluster/job/result.rb,
lib/arachni/browser_cluster/jobs/taint_trace.rb,
lib/arachni/browser_cluster/jobs/dom_exploration.rb,
lib/arachni/browser_cluster/jobs/browser_provider.rb,
lib/arachni/browser_cluster/jobs/taint_trace/result.rb,
lib/arachni/browser_cluster/jobs/dom_exploration/result.rb,
lib/arachni/browser_cluster/jobs/taint_trace/event_trigger.rb,
lib/arachni/browser_cluster/jobs/dom_exploration/event_trigger.rb,
lib/arachni/browser_cluster/jobs/taint_trace/event_trigger/result.rb,
lib/arachni/browser_cluster/jobs/dom_exploration/event_trigger/result.rb

Overview

Author:

  • Tasos “Zapotek” Laskos <tasos.laskos@arachni-scanner.com>

Defined Under Namespace

Modules: Jobs Classes: Error, Job, Worker

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Utilities

#available_port, available_port_mutex, #bytes_to_kilobytes, #bytes_to_megabytes, #caller_name, #caller_path, #cookie_decode, #cookie_encode, #cookies_from_file, #cookies_from_parser, #cookies_from_response, #exception_jail, #exclude_path?, #follow_protocol?, #form_decode, #form_encode, #forms_from_parser, #forms_from_response, #full_and_absolute_url?, #generate_token, #get_path, #hms_to_seconds, #html_decode, #html_encode, #include_path?, #links_from_parser, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_set_cookie, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #random_seed, #redundant_path?, #regexp_array_match, #remove_constants, #request_parse_body, #seconds_to_hms, #skip_page?, #skip_path?, #skip_resource?, #skip_response?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parse_query, #uri_parser, #uri_rewrite

Methods included from UI::Output

#debug?, #debug_level_1?, #debug_level_2?, #debug_level_3?, #debug_level_4?, #debug_off, #debug_on, #disable_only_positives, #included, #mute, #muted?, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_level_1, #print_debug_level_2, #print_debug_level_3, #print_debug_level_4, #print_error, #print_error_backtrace, #print_exception, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #unmute, #verbose?, #verbose_on

Constructor Details

#initialize(options = {}) ⇒ BrowserCluster

Returns a new instance of BrowserCluster.

Parameters:

  • options (Hash) (defaults to: {})

Options Hash (options):

  • :pool_size (Integer) — default: 5 —

    Amount of browsers to add to the pool.

  • :time_to_live (Integer) — default: 10 —

    Restricts each browser's lifetime to the given amount of pages. When that number is exceeded the current process is killed and a new one is pushed to the pool. Helps prevent memory leak issues.

Raises:

  • ArgumentError On missing `:handler` option.



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/arachni/browser_cluster.rb', line 75

def initialize( options = {} )
    super()

    {
        pool_size: Options.browser_cluster.pool_size
    }.merge( options ).each do |k, v|
        begin
            send( "#{k}=", try_dup( v ) )
        rescue NoMethodError
            instance_variable_set( "@#{k}".to_sym, v )
        end
    end

    # Used to sync operations between workers per Job#id.
    @skip_states_per_job = {}

    # Callbacks for each job per Job#id. We need to keep track of this
    # here because jobs are serialized and off-loaded to disk and thus can't
    # contain Block or Proc objects.
    @job_callbacks = {}

    # Keeps track of the amount of pending jobs distributed across the
    # cluster, by Job#id. Once a job's count reaches 0, it's passed to
    # #job_done.
    @pending_jobs = Hash.new(0)
    @pending_job_counter = 0

    # Jobs are off-loaded to disk.
    @jobs = Support::Database::Queue.new
    @jobs.max_buffer_size = 10

    # Worker pool holding BrowserCluster::Worker instances.
    @workers     = []

    @mutex       = Monitor.new
    @done_signal = Queue.new

    initialize_workers
end

Instance Attribute Details

#pending_job_counter ⇒ Integer (readonly)

Returns Number of pending jobs.

Returns:

  • (Integer) —

    Number of pending jobs.



64
65
66
 
# File 'lib/arachni/browser_cluster.rb', line 64

def pending_job_counter
  @pending_job_counter
end

#pool_size ⇒ Integer (readonly)

Returns Amount of browser instances in the pool.

Returns:

  • (Integer) —

    Amount of browser instances in the pool.



56
57
58
 
# File 'lib/arachni/browser_cluster.rb', line 56

def pool_size
  @pool_size
end

#workers ⇒ Array<Worker> (readonly)

Returns Worker pool.

Returns:



60
61
62
 
# File 'lib/arachni/browser_cluster.rb', line 60

def workers
  @workers
end

Class Method Details

.add_to_total_job_time(time) ⇒ Object 



438
439
440
441
 
# File 'lib/arachni/browser_cluster.rb', line 438

def self.add_to_total_job_time( time )
    @total_job_time ||= 0.0
    @total_job_time += time.to_f
end

.completed_job_count ⇒ Object 



430
431
432
 
# File 'lib/arachni/browser_cluster.rb', line 430

def self.completed_job_count
    @completed_job_count.to_i
end

.increment_completed_job_count ⇒ Object 



420
421
422
423
 
# File 'lib/arachni/browser_cluster.rb', line 420

def self.increment_completed_job_count
    @completed_job_count ||= 0
    @completed_job_count += 1
end

.increment_queued_job_count ⇒ Object 



415
416
417
418
 
# File 'lib/arachni/browser_cluster.rb', line 415

def self.increment_queued_job_count
    @queued_job_count ||= 0
    @queued_job_count += 1
end

.increment_time_out_count ⇒ Object 



425
426
427
428
 
# File 'lib/arachni/browser_cluster.rb', line 425

def self.increment_time_out_count
    @time_out_count ||= 0
    @time_out_count += 1
end

.seconds_per_job ⇒ Object 



410
411
412
413
 
# File 'lib/arachni/browser_cluster.rb', line 410

def self.seconds_per_job
    n = (total_job_time / Float( completed_job_count ))
    n.nan? ? 0 : n
end

.statistics ⇒ Object 



443
444
445
446
447
448
449
450
451
 
# File 'lib/arachni/browser_cluster.rb', line 443

def self.statistics
    {
        seconds_per_job:     seconds_per_job,
        total_job_time:      total_job_time,
        queued_job_count:    @queued_job_count    || 0,
        completed_job_count: @completed_job_count || 0,
        time_out_count:      @time_out_count      || 0
    }
end

.total_job_time ⇒ Object 



434
435
436
 
# File 'lib/arachni/browser_cluster.rb', line 434

def self.total_job_time
    @total_job_time.to_i
end

Instance Method Details

#add_to_total_job_time(time) ⇒ Object 



404
405
406
407
408
 
# File 'lib/arachni/browser_cluster.rb', line 404

def add_to_total_job_time( time )
    synchronize do
        self.class.add_to_total_job_time( time )
    end
end

#callback_for(job) ⇒ Object 



382
383
384
 
# File 'lib/arachni/browser_cluster.rb', line 382

def callback_for( job )
    @job_callbacks[job.id]
end

#done? ⇒ Bool

Returns `true` if there are no resources to analyze and no running workers.

Returns:

  • (Bool) —

    `true` if there are no resources to analyze and no running workers.



275
276
277
278
 
# File 'lib/arachni/browser_cluster.rb', line 275

def done?
    fail_if_shutdown
    synchronize { @pending_job_counter == 0 }
end

#explore(resource, options = {}, cb = nil, &block) ⇒ Object

Parameters:

See Also:



182
183
184
185
186
187
188
 
# File 'lib/arachni/browser_cluster.rb', line 182

def explore( resource, options = {}, cb = nil, &block )
    queue(
        Jobs::DOMExploration.new( options.merge( resource: resource ) ),
        cb,
        &block
    )
end

#handle_job_result(result) ⇒ Object

Parameters:



254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
 
# File 'lib/arachni/browser_cluster.rb', line 254

def handle_job_result( result )
    return if @shutdown
    return if job_done? result.job

    synchronize do
        print_debug "Got job result: #{result}"

        exception_jail( false ) do
            @job_callbacks[result.job.id].call( *[
                result,
                result.job.args,
                self
            ].flatten.compact)
        end
    end

    nil
end

#increment_completed_job_count ⇒ Object 



392
393
394
395
396
 
# File 'lib/arachni/browser_cluster.rb', line 392

def increment_completed_job_count
    synchronize do
        self.class.increment_completed_job_count
    end
end

#increment_queued_job_count ⇒ Object 



386
387
388
389
390
 
# File 'lib/arachni/browser_cluster.rb', line 386

def increment_queued_job_count
    synchronize do
        self.class.increment_queued_job_count
    end
end

#increment_time_out_count ⇒ Object 



398
399
400
401
402
 
# File 'lib/arachni/browser_cluster.rb', line 398

def increment_time_out_count
    synchronize do
        self.class.increment_time_out_count
    end
end

#javascript_token ⇒ String

Returns Javascript token used to namespace the custom JS environment.

Returns:

  • (String) —

    Javascript token used to namespace the custom JS environment.



117
118
119
 
# File 'lib/arachni/browser_cluster.rb', line 117

def javascript_token
    Browser::Javascript::TOKEN
end

#job_done(job) ⇒ Object

Parameters:

  • job (Job) —

    Job to mark as done. Will remove any callbacks and associated Worker states.



211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/arachni/browser_cluster.rb', line 211

def job_done( job )
    synchronize do
        print_debug "Job done: #{job}"

        @pending_job_counter  -= 1
        @pending_jobs[job.id] -= 1

        increment_completed_job_count
        add_to_total_job_time( job.time )

        notify_on_job_done job

        if !job.never_ending?
            @skip_states_per_job.delete job.id
            @job_callbacks.delete job.id
        end

        if @pending_job_counter == 0
            print_debug_level_2 'Pending job counter reached 0.'
            @done_signal << nil
        end
    end
end

#job_done?(job, fail_if_not_found = true) ⇒ Bool

Returns `true` if the `job` has been marked as finished, `false` otherwise.

Parameters:

Returns:

  • (Bool) —

    `true` if the `job` has been marked as finished, `false` otherwise.

Raises:



241
242
243
244
245
246
247
248
249
 
# File 'lib/arachni/browser_cluster.rb', line 241

def job_done?( job, fail_if_not_found = true )
    return false if job.never_ending?

    synchronize do
        fail_if_job_not_found job if fail_if_not_found
        return false if !@pending_jobs.include?( job.id )
        @pending_jobs[job.id] == 0
    end
end

#pop ⇒ Job

Returns Pops a job from the queue.

Returns:

  • (Job) —

    Pops a job from the queue.

See Also:



329
330
331
332
333
334
335
336
337
 
# File 'lib/arachni/browser_cluster.rb', line 329

def pop
    print_debug 'Popping...'
    {} while job_done?( job = @jobs.pop )
    print_debug "...popped: #{job}"

    notify_on_pop job

    job
end

#queue(job, cb = nil, &block) ⇒ Object

Parameters:

Raises:



140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
 
# File 'lib/arachni/browser_cluster.rb', line 140

def queue( job, cb = nil, &block )
    fail_if_shutdown
    fail_if_job_done job

    @done_signal.clear

    synchronize do
        print_debug "Queueing: #{job}"

        notify_on_queue job

        self.class.increment_queued_job_count

        @pending_job_counter  += 1
        @pending_jobs[job.id] += 1

        if cb
            @job_callbacks[job.id] = cb
        elsif block
            @job_callbacks[job.id] = block
        end

        if !@job_callbacks[job.id]
            fail ArgumentError, "No callback set for job ID #{job.id}."
        end

        @jobs << job
    end

    nil
end

#shutdown(wait = true) ⇒ Object

Shuts the cluster down.



296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
 
# File 'lib/arachni/browser_cluster.rb', line 296

def shutdown( wait = true )
    print_debug 'Shutting down...'
    @shutdown = true

    print_debug_level_2 'Clearing jobs...'
    # Clear the jobs -- don't forget this, it also removes the disk files for
    # the contained items.
    @jobs.clear
    print_debug_level_2 '...done.'

    print_debug_level_2 "Shutting down #{@workers.size} workers..."
    # Kill the browsers.
    @workers.each { |b| exception_jail( false ) { b.shutdown wait } }
    @workers.clear
    print_debug_level_2 '...done.'

    print_debug_level_2 'Clearing data and state...'
    # Very important to leave these for last, they may contain data
    # necessary to cleanly handle interrupted jobs.
    @job_callbacks.clear
    @skip_states_per_job.clear
    @pending_jobs.clear
    print_debug_level_2 '...done.'

    print_debug '...shutdown complete.'
    true
end

#skip_state(job_id, state) ⇒ Object

Used to sync operations between browser workers.

Parameters:

  • job_id (Integer) —

    Job ID.

  • state (String) —

    State to skip in the future.



364
365
366
 
# File 'lib/arachni/browser_cluster.rb', line 364

def skip_state( job_id, state )
    synchronize { skip_states( job_id ) << state }
end

#skip_state?(job_id, state) ⇒ Boolean

Used to sync operations between browser workers.

Parameters:

  • job_id (Integer) —

    Job ID.

  • state (String) —

    Should the given state be skipped?

Returns:

  • (Boolean)

Raises:



350
351
352
353
354
 
# File 'lib/arachni/browser_cluster.rb', line 350

def skip_state?( job_id, state )
    synchronize do
        skip_states( job_id ).include? state
    end
end

#skip_states(id) ⇒ Object 



374
375
376
377
378
379
 
# File 'lib/arachni/browser_cluster.rb', line 374

def skip_states( id )
    synchronize do
        @skip_states_per_job[id] ||=
            Support::LookUp::HashSet.new( hasher: :persistent_hash )
    end
end

#trace_taint(resource, options = {}, cb = nil, &block) ⇒ Object

Parameters:

See Also:



200
201
202
203
204
205
206
 
# File 'lib/arachni/browser_cluster.rb', line 200

def trace_taint( resource, options = {}, cb = nil, &block )
    queue(
        Jobs::TaintTrace.new( options.merge( resource: resource ) ),
        cb,
        &block
    )
end

#update_skip_states(id, lookups) ⇒ Object 



369
370
371
 
# File 'lib/arachni/browser_cluster.rb', line 369

def update_skip_states( id, lookups )
    synchronize { skip_states( id ).merge lookups }
end

#wait ⇒ Object

Blocks until all resources have been analyzed.



285
286
287
288
289
290
291
292
293
 
# File 'lib/arachni/browser_cluster.rb', line 285

def wait
    fail_if_shutdown

    print_debug 'Waiting to finish...'
    @done_signal.pop if !done?
    print_debug '...finish.'

    self
end

#with_browser(*args, &block) ⇒ Object

Note:

Operates in non-blocking mode.

Parameters:

  • block (Block) —

    Block to which to pass a Worker as soon as one is available.



125
126
127
128
129
130
131
132
 
# File 'lib/arachni/browser_cluster.rb', line 125

def with_browser( *args, &block )
    method_handler = nil
    if args.last.is_a? Method
        method_handler = args.pop
    end

    queue( Jobs::BrowserProvider.new( args ), method_handler, &block )
end