Class: Arachni::HTTP::Client

Inherits:
Object
  • Object
show all
Includes:
Support::Mixins::Observable, UI::Output, Utilities, Singleton
Defined in:
lib/arachni/http/client.rb,
lib/arachni/http/client/dynamic_404_handler.rb

Overview

Provides a system-wide, simple and high-performance HTTP client.

Author:

  • Tasos “Zapotek” Laskos <tasos.laskos@arachni-scanner.com>

Defined Under Namespace

Classes: Dynamic404Handler, Error

Constant Summary collapse

MAX_CONCURRENCY =

Default maximum concurrency for HTTP requests.

20
HTTP_TIMEOUT =

Default 1 minute timeout for HTTP requests.

60_000
SEED_HEADER_NAME = 
'X-Arachni-Scan-Seed'

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Support::Mixins::Observable

included

Methods included from Utilities

#available_port, available_port_mutex, #bytes_to_kilobytes, #bytes_to_megabytes, #caller_name, #caller_path, #cookie_decode, #cookie_encode, #cookies_from_file, #cookies_from_parser, #cookies_from_response, #exception_jail, #exclude_path?, #follow_protocol?, #form_decode, #form_encode, #forms_from_parser, #forms_from_response, #full_and_absolute_url?, #generate_token, #get_path, #hms_to_seconds, #html_decode, #html_encode, #include_path?, #links_from_parser, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_set_cookie, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #random_seed, #redundant_path?, #regexp_array_match, #remove_constants, #request_parse_body, #seconds_to_hms, #skip_page?, #skip_path?, #skip_resource?, #skip_response?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parse_query, #uri_parser, #uri_rewrite

Methods included from UI::Output

#debug?, #debug_level_1?, #debug_level_2?, #debug_level_3?, #debug_level_4?, #debug_off, #debug_on, #disable_only_positives, #included, #mute, #muted?, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_level_1, #print_debug_level_2, #print_debug_level_3, #print_debug_level_4, #print_error, #print_error_backtrace, #print_exception, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #unmute, #verbose?, #verbose_on

Constructor Details

#initialize ⇒ Client

Returns a new instance of Client.



123
124
125
126
 
# File 'lib/arachni/http/client.rb', line 123

def initialize
    super
    reset
end

Instance Attribute Details

#burst_response_count ⇒ Integer (readonly)

Returns Amount of responses received for the running requests (of the current burst).

Returns:

  • (Integer) —

    Amount of responses received for the running requests (of the current burst).



116
117
118
 
# File 'lib/arachni/http/client.rb', line 116

def burst_response_count
  @burst_response_count
end

#burst_response_time_sum ⇒ Integer (readonly)

Returns Sum of the response times for the running requests (of the current burst).

Returns:

  • (Integer) —

    Sum of the response times for the running requests (of the current burst).



112
113
114
 
# File 'lib/arachni/http/client.rb', line 112

def burst_response_time_sum
  @burst_response_time_sum
end

#dynamic_404_handler ⇒ Dynamic404Handler (readonly)

Returns:



119
120
121
 
# File 'lib/arachni/http/client.rb', line 119

def dynamic_404_handler
  @dynamic_404_handler
end

#headers ⇒ Hash (readonly)

Returns Default headers for requests.

Returns:



96
97
98
 
# File 'lib/arachni/http/client.rb', line 96

def headers
  @headers
end

#original_max_concurrency ⇒ Object (readonly)

Returns the value of attribute original_max_concurrency.



121
122
123
 
# File 'lib/arachni/http/client.rb', line 121

def original_max_concurrency
  @original_max_concurrency
end

#request_count ⇒ Integer (readonly)

Returns Amount of performed requests.

Returns:

  • (Integer) —

    Amount of performed requests.



100
101
102
 
# File 'lib/arachni/http/client.rb', line 100

def request_count
  @request_count
end

#response_count ⇒ Integer (readonly)

Returns Amount of received responses.

Returns:

  • (Integer) —

    Amount of received responses.



104
105
106
 
# File 'lib/arachni/http/client.rb', line 104

def response_count
  @response_count
end

#time_out_count ⇒ Integer (readonly)

Returns Amount of timed-out requests.

Returns:

  • (Integer) —

    Amount of timed-out requests.



108
109
110
 
# File 'lib/arachni/http/client.rb', line 108

def time_out_count
  @time_out_count
end

#url ⇒ String (readonly)

Returns Framework target URL, used as reference.

Returns:

  • (String) —

    Framework target URL, used as reference.



92
93
94
 
# File 'lib/arachni/http/client.rb', line 92

def url
  @url
end

Class Method Details

.method_missing(sym, *args, &block) ⇒ Object 



501
502
503
 
# File 'lib/arachni/http/client.rb', line 501

def self.method_missing( sym, *args, &block )
    instance.send( sym, *args, &block )
end

Instance Method Details

#abort ⇒ Object

Aborts the running requests on a best effort basis.



278
279
280
 
# File 'lib/arachni/http/client.rb', line 278

def abort
    exception_jail { client_abort }
end

#after_each_run(&block) ⇒ Arachni::HTTP

Returns self.

Parameters:

  • block (Block) —

    Called after each #run.

Returns:



58
 
# File 'lib/arachni/http/client.rb', line 58

advertise :after_each_run

#after_run(&block) ⇒ Arachni::HTTP::Client

Returns `self`.

Parameters:

  • block (Block) —

    Called after the next #run.

Returns:



50
 
# File 'lib/arachni/http/client.rb', line 50

advertise :after_run

#burst_average_response_time ⇒ Float

Returns Average response time for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Average response time for the running requests (i.e. the current burst).



313
314
315
316
 
# File 'lib/arachni/http/client.rb', line 313

def burst_average_response_time
    return 0 if @burst_response_count == 0
    @burst_response_time_sum / Float( @burst_response_count )
end

#burst_responses_per_second ⇒ Float

Returns Responses/second for the running requests (i.e. the current burst).

Returns:

  • (Float) —

    Responses/second for the running requests (i.e. the current burst).



320
321
322
323
324
325
 
# File 'lib/arachni/http/client.rb', line 320

def burst_responses_per_second
    if @async_burst_response_count > 0 && burst_runtime > 0
        return @async_burst_response_count / burst_runtime
    end
    0
end

#burst_runtime ⇒ Float

Returns Amount of time (in seconds) that the current burst has been running.

Returns:

  • (Float) —

    Amount of time (in seconds) that the current burst has been running.



306
307
308
309
 
# File 'lib/arachni/http/client.rb', line 306

def burst_runtime
    @burst_runtime.to_i > 0 ?
        @burst_runtime : Time.now - (@burst_runtime_start || Time.now)
end

#cookie(url = @url, options = {}, &block) ⇒ Request, Response

Performs a `GET` request sending the cookies in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



458
459
460
461
 
# File 'lib/arachni/http/client.rb', line 458

def cookie( url = @url, options = {}, &block )
    options[:cookies] = (options.delete( :parameters ) || {}).dup
    request( url, options, &block )
end

#cookie_jar ⇒ CookieJar

Returns:



210
211
212
 
# File 'lib/arachni/http/client.rb', line 210

def cookie_jar
    State.http.cookie_jar
end

#cookies ⇒ Array<Arachni::Element::Cookie>

Returns All cookies in the jar.

Returns:



341
342
343
 
# File 'lib/arachni/http/client.rb', line 341

def cookies
    cookie_jar.cookies
end

#get(url = @url, options = {}, &block) ⇒ Request, Response

Performs a `GET` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



426
427
428
 
# File 'lib/arachni/http/client.rb', line 426

def get( url = @url, options = {}, &block )
    request( url, options, &block )
end

#header(url = @url, options = {}, &block) ⇒ Request, Response

Performs a `GET` request sending the headers in `:parameters`.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



469
470
471
472
473
 
# File 'lib/arachni/http/client.rb', line 469

def header( url = @url, options = {}, &block )
    options[:headers] ||= {}
    options[:headers].merge!( (options.delete( :parameters ) || {}).dup )
    request( url, options, &block )
end

#inspect ⇒ Object 



505
506
507
508
509
 
# File 'lib/arachni/http/client.rb', line 505

def inspect
    s = "#<#{self.class} "
    statistics.each { |k, v| s << "@#{k}=#{v.inspect} " }
    s << '>'
end

#max_concurrency ⇒ Integer

Returns Current maximum concurrency of HTTP requests.

Returns:

  • (Integer) —

    Current maximum concurrency of HTTP requests.



335
336
337
 
# File 'lib/arachni/http/client.rb', line 335

def max_concurrency
    @hydra.max_concurrency
end

#max_concurrency=(concurrency) ⇒ Object

Parameters:

  • concurrency (Integer) —

    Sets the maximum concurrency of HTTP requests.



329
330
331
 
# File 'lib/arachni/http/client.rb', line 329

def max_concurrency=( concurrency )
    @hydra.max_concurrency = concurrency
end

#on_complete(&block) ⇒ Object 



70
 
# File 'lib/arachni/http/client.rb', line 70

advertise :on_complete

#on_new_cookies(&block) ⇒ Object

Parameters:

  • block (Block) —

    To be passed the new cookies and the response that set them



67
 
# File 'lib/arachni/http/client.rb', line 67

advertise :on_new_cookies

#on_queue(&block) ⇒ Object 



61
 
# File 'lib/arachni/http/client.rb', line 61

advertise :on_queue

#parse_and_set_cookies(response) ⇒ Object

Note:

Runs #on_new_cookies callbacks.

Parameters:

  • response (Response) —

    Extracts cookies from `response` and updates the cookie-jar.



494
495
496
497
498
499
 
# File 'lib/arachni/http/client.rb', line 494

def parse_and_set_cookies( response )
    cookies = Cookie.from_response( response )
    update_cookies( cookies )

    notify_on_new_cookies( cookies, response )
end

#post(url = @url, options = {}, &block) ⇒ Request, Response

Performs a `POST` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



436
437
438
439
 
# File 'lib/arachni/http/client.rb', line 436

def post( url = @url, options = {}, &block )
    options[:body] = (options.delete( :parameters ) || {}).dup
    request( url, options.merge( method: :post ), &block )
end

#queue(request) ⇒ Object

Parameters:

  • request (Request) —

    Request to queue.



477
478
479
480
 
# File 'lib/arachni/http/client.rb', line 477

def queue( request )
    notify_on_queue( request )
    forward_request( request )
end

#request(url = @url, options = {}, &block) ⇒ Request, Response

Queues/performs a generic request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Options Hash (options):

  • :cookies (Hash) — default: {} —

    Extra cookies to use for this request.

  • :no_cookie_jar (Hash) — default: false —

    Do not include cookies from the #cookie_jar.

Returns:



360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
 
# File 'lib/arachni/http/client.rb', line 360

def request( url = @url, options = {}, &block )
    fail ArgumentError, 'URL cannot be empty.' if !url

    options     = options.dup
    cookies     = options.delete( :cookies )     || {}
    raw_cookies = options.delete( :raw_cookies ) || []
    raw_cookie_names = Set.new( raw_cookies.map(&:name) )

    exception_jail false do
        if !options.delete( :no_cookie_jar )
            raw_cookies |= begin
                cookie_jar.for_url( url ).reject do |c|
                    cookies.include?( c.name ) ||
                        raw_cookie_names.include?( c.name )
                end
            rescue => e
                print_error "Could not get cookies for URL '#{url}' from Cookiejar (#{e})."
                print_error_backtrace e
                []
            end
        end

        on_headers    = options.delete(:on_headers)
        on_body       = options.delete(:on_body)
        on_body_line  = options.delete(:on_body_line)
        on_body_lines = options.delete(:on_body_lines)

        request = Request.new( options.merge(
            url:         url,
            headers:     headers.merge( options.delete( :headers ) || {}, false ),
            cookies:     cookies,
            raw_cookies: raw_cookies
        ))

        if on_headers
            request.on_headers( &on_headers )
        end

        if on_body
            request.on_body( &on_body )
        end

        if on_body_line
            request.on_body_line( &on_body_line )
        end

        if on_body_lines
            request.on_body_lines( &on_body_lines )
        end

        if block_given?
            request.on_complete( &block )
        end

        queue( request )
        return request.run if request.blocking?
        request
    end
end

#reset(hooks_too = true) ⇒ Arachni::HTTP

Returns Reset `self`.

Returns:



145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/arachni/http/client.rb', line 145

def reset( hooks_too = true )
    clear_observers if hooks_too
    State.http.clear

    @url = Options.url.to_s
    @url = nil if @url.empty?

    client_initialize

    reset_options

    if Options.http.cookie_jar_filepath
        cookie_jar.load( Options.http.cookie_jar_filepath )
    end

    Options.http.cookies.each do |name, value|
        update_cookies( name => value )
    end

    if Options.http.cookie_string
        update_cookies( Options.http.cookie_string )
    end

    reset_burst_info

    @request_count  = 0
    @async_response_count = 0
    @response_count = 0
    @time_out_count = 0

    @total_response_time_sum = 0
    @total_runtime           = 0

    @queue_size = 0

    @dynamic_404_handler = Dynamic404Handler.new

    self
end

#reset_options ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/arachni/http/client.rb', line 128

def reset_options
    @original_max_concurrency = Options.http.request_concurrency || MAX_CONCURRENCY
    self.max_concurrency      = @original_max_concurrency

    headers.clear
    headers.merge!(
        'Accept'              => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'User-Agent'          => Options.http.user_agent,
        'Accept-Language'     => 'en-US,en;q=0.8,he;q=0.6',
        SEED_HEADER_NAME      => Arachni::Utilities.random_seed
    )
    headers['From'] = Options.authorized_by if Options.authorized_by
    headers.merge!( Options.http.request_headers, false )
end

#run ⇒ Object

Runs all queued requests



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/arachni/http/client.rb', line 219

def run
    exception_jail false do
        @burst_runtime = nil

        begin
            run_and_update_statistics

            duped_after_run = observers_for( :after_run ).dup
            observers_for( :after_run ).clear
            duped_after_run.each { |block| block.call }
        end while @queue_size > 0 || observers_for( :after_run ).any?

        notify_after_each_run

        # Prune the custom 404 cache after callbacks have been called.
        @dynamic_404_handler.prune

        @curr_res_time = 0
        @curr_res_cnt  = 0

        true
    end
end

#sandbox(&block) ⇒ Object

Note:

Cookies or new callbacks set as a result of the block won't affect the HTTP singleton.

Return value of the block.

Parameters:

  • block (Block) —

    Block to executes inside a sandbox.

Returns:

  • (Object) —

    Return value of the block.



251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/arachni/http/client.rb', line 251

def sandbox( &block )
    h = {}
    instance_variables.each do |iv|
        val = instance_variable_get( iv )
        h[iv] = val.deep_clone rescue val.dup rescue val
    end

    saved_observers = dup_observers

    pre_cookies = cookies.deep_clone
    pre_headers = headers.deep_clone

    ret = block.call( self )

    cookie_jar.clear
    update_cookies pre_cookies

    headers.clear
    headers.merge! pre_headers

    h.each { |iv, val| instance_variable_set( iv, val ) }
    set_observers( saved_observers )

    ret
end

#statistics ⇒ Hash

Returns Hash including HTTP client statistics including:

Returns:



200
201
202
203
204
205
206
207
 
# File 'lib/arachni/http/client.rb', line 200

def statistics
   [:request_count, :response_count, :time_out_count,
    :total_responses_per_second, :burst_response_time_sum,
    :burst_response_count, :burst_responses_per_second,
    :burst_average_response_time, :total_average_response_time,
    :max_concurrency, :original_max_concurrency].
       inject({}) { |h, k| h[k] = send(k); h }
end

#total_average_response_time ⇒ Float

Returns Average response time for all requests.

Returns:

  • (Float) —

    Average response time for all requests.



291
292
293
294
 
# File 'lib/arachni/http/client.rb', line 291

def total_average_response_time
    return 0 if @response_count == 0
    @total_response_time_sum / Float( @response_count )
end

#total_responses_per_second ⇒ Float

Returns Responses/second.

Returns:

  • (Float) —

    Responses/second.



297
298
299
300
301
302
 
# File 'lib/arachni/http/client.rb', line 297

def total_responses_per_second
    if @async_response_count > 0 && total_runtime > 0
        return @async_response_count / Float( total_runtime )
    end
    0
end

#total_runtime ⇒ Integer

Returns Amount of time (in seconds) that has been devoted to performing requests and getting responses.

Returns:

  • (Integer) —

    Amount of time (in seconds) that has been devoted to performing requests and getting responses.



285
286
287
 
# File 'lib/arachni/http/client.rb', line 285

def total_runtime
    @total_runtime > 0 ? @total_runtime : burst_runtime
end

#trace(url = @url, options = {}, &block) ⇒ Request, Response

Performs a `TRACE` request.

Parameters:

  • url (String) (defaults to: @url) —

    URL to request.

  • options (Hash) (defaults to: {}) —

    Request options with the following extras:

  • block (Block) —

    Callback to be passed the response.

Returns:

See Also:



447
448
449
 
# File 'lib/arachni/http/client.rb', line 447

def trace( url = @url, options = {}, &block )
    request( url, options.merge( method: :trace ), &block )
end

#update_cookies(cookies) ⇒ Object Also known as: set_cookies

Parameters:



484
485
486
487
 
# File 'lib/arachni/http/client.rb', line 484

def update_cookies( cookies )
    cookie_jar.update( cookies )
    cookie_jar.cookies
end