Class: Arachni::HTTP::Request

Inherits:
Message show all
Includes:
UI::Output, Utilities
Defined in:
lib/arachni/http/request.rb,
lib/arachni/http/request/scope.rb

Overview

HTTP Request representation.

Author:

  • Tasos “Zapotek” Laskos <tasos.laskos@arachni-scanner.com>

Defined Under Namespace

Classes: Scope

Constant Summary collapse

ENCODE_CACHE = 
Support::Cache::LeastRecentlyPushed.new( 1_000 )
REDIRECT_LIMIT =

Default redirect limit, RFC says 5 max.

5
MODES =

Supported modes of operation.

[
    # Asynchronous (non-blocking) (Default)
    :async,

    # Synchronous (blocking)
    :sync
]

Instance Attribute Summary collapse

Attributes inherited from Message

#body, #headers, #url

Class Method Summary collapse

Instance Method Summary collapse

Methods included from UI::Output

#debug?, #debug_level_1?, #debug_level_2?, #debug_level_3?, #debug_level_4?, #debug_off, #debug_on, #disable_only_positives, #included, #mute, #muted?, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_level_1, #print_debug_level_2, #print_debug_level_3, #print_debug_level_4, #print_error, #print_error_backtrace, #print_exception, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #unmute, #verbose?, #verbose_on

Methods included from Utilities

#available_port, available_port_mutex, #bytes_to_kilobytes, #bytes_to_megabytes, #caller_name, #caller_path, #cookie_decode, #cookie_encode, #cookies_from_file, #cookies_from_parser, #cookies_from_response, #exception_jail, #exclude_path?, #follow_protocol?, #form_decode, #form_encode, #forms_from_parser, #forms_from_response, #full_and_absolute_url?, #generate_token, #get_path, #hms_to_seconds, #html_decode, #html_encode, #include_path?, #links_from_parser, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_set_cookie, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #random_seed, #redundant_path?, #regexp_array_match, #remove_constants, #request_parse_body, #seconds_to_hms, #skip_page?, #skip_path?, #skip_resource?, #skip_response?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parse_query, #uri_parser, #uri_rewrite

Methods inherited from Message

#parsed_url, #scope, #update

Constructor Details

#initialize(options = {}) ⇒ Request

Returns a new instance of Request.

Parameters:

  • options (Hash) (defaults to: {}) —

    Request options.

Options Hash (options):

  • :url (String) —

    URL.

  • :parameters (Hash) — default: {} —

    Request parameters.

  • :body (String) — default: {} —

    Request body.

  • :train (Bool) — default: false —

    Force Arachni to analyze the response looking for new elements.

  • :mode (Symbol) — default: :async —

    Mode in which to perform the request:

    • `:async` – Asynchronous (non-blocking) (Default).

    • `:sync` – Synchronous (blocking).

  • :headers (Hash) — default: {} —

    Extra HTTP request headers.

  • :cookies (Hash) — default: {} —

    Cookies for the request.



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/arachni/http/request.rb', line 141

def initialize( options = {} )
    options[:method] ||= :get

    super( options )

    @train           = false if @train.nil?
    @fingerprint     = true  if @fingerprint.nil?
    @update_cookies  = false if @update_cookies.nil?
    @follow_location = false if @follow_location.nil?
    @max_redirects   = (Options.http.request_redirect_limit || REDIRECT_LIMIT)

    @on_headers    = []
    @on_body       = []
    @on_body_line  = []
    @on_body_lines = []
    @on_complete   = []

    @raw_parameters ||= []
    @timeout        ||= Options.http.request_timeout
    @mode           ||= :async
    @parameters     ||= {}
    @cookies        ||= {}
    @raw_cookies    ||= []
end

Instance Attribute Details

#cookies ⇒ Hash

Returns Cookies set for this request.

Returns:

  • (Hash) —

    Cookies set for this request.



67
68
69
 
# File 'lib/arachni/http/request.rb', line 67

def cookies
  @cookies
end

#effective_body ⇒ String

Note:

Available only via completed Arachni::HTTP::Response#request.

Returns Transmitted HTTP request body.

Returns:

  • (String) —

    Transmitted HTTP request body.



88
89
90
 
# File 'lib/arachni/http/request.rb', line 88

def effective_body
  @effective_body
end

#follow_location ⇒ Bool

Returns Follow `Location` headers.

Returns:

  • (Bool) —

    Follow `Location` headers.



49
50
51
 
# File 'lib/arachni/http/request.rb', line 49

def follow_location
  @follow_location
end

#headers_string ⇒ String

Note:

Available only via completed Arachni::HTTP::Response#request.

Returns Transmitted HTTP request headers.

Returns:

  • (String) —

    Transmitted HTTP request headers.



82
83
84
 
# File 'lib/arachni/http/request.rb', line 82

def headers_string
  @headers_string
end

#high_priority ⇒ Bool

Returns:

  • (Bool) 


106
107
108
 
# File 'lib/arachni/http/request.rb', line 106

def high_priority
  @high_priority
end

#id ⇒ Integer

Returns Auto-incremented ID for this request (set by Client#request).

Returns:

  • (Integer) —

    Auto-incremented ID for this request (set by Client#request).



37
38
39
 
# File 'lib/arachni/http/request.rb', line 37

def id
  @id
end

#max_redirects ⇒ Integer

Returns Maximum number of redirects to follow.

Returns:

  • (Integer) —

    Maximum number of redirects to follow.

See Also:



55
56
57
 
# File 'lib/arachni/http/request.rb', line 55

def max_redirects
  @max_redirects
end

#mode ⇒ Symbol

Returns Mode of operation for the request.

Returns:

  • (Symbol) —

    Mode of operation for the request.

See Also:



76
77
78
 
# File 'lib/arachni/http/request.rb', line 76

def mode
  @mode
end

#parameters ⇒ Hash

Returns Request parameters.

Returns:

  • (Hash) —

    Request parameters.



41
42
43
 
# File 'lib/arachni/http/request.rb', line 41

def parameters
  @parameters
end

#password ⇒ String

Returns HTTP password.

Returns:



63
64
65
 
# File 'lib/arachni/http/request.rb', line 63

def password
  @password
end

#performer ⇒ Object

Entity which performed the request – mostly used to track which response was a result of which submitted element.



92
93
94
 
# File 'lib/arachni/http/request.rb', line 92

def performer
  @performer
end

#proxy ⇒ String

Returns `host:port`.

Returns:



96
97
98
 
# File 'lib/arachni/http/request.rb', line 96

def proxy
  @proxy
end

#proxy_type ⇒ String

Returns:



103
104
105
 
# File 'lib/arachni/http/request.rb', line 103

def proxy_type
  @proxy_type
end

#proxy_user_password ⇒ String

Returns `user:password`.

Returns:

  • (String) —

    `user:password`



100
101
102
 
# File 'lib/arachni/http/request.rb', line 100

def proxy_user_password
  @proxy_user_password
end

#raw_cookies ⇒ Array<Element::Cookie> (readonly)

Returns:



70
71
72
 
# File 'lib/arachni/http/request.rb', line 70

def raw_cookies
  @raw_cookies
end

#raw_parameters ⇒ Array

Returns Parameters which should not be encoded, by name.

Returns:

  • (Array) —

    Parameters which should not be encoded, by name.



114
115
116
 
# File 'lib/arachni/http/request.rb', line 114

def raw_parameters
  @raw_parameters
end

#response ⇒ Response

Returns:



117
118
119
 
# File 'lib/arachni/http/request.rb', line 117

def response
  @response
end

#response_body_buffer ⇒ Object 



120
121
122
 
# File 'lib/arachni/http/request.rb', line 120

def response_body_buffer
  @response_body_buffer
end

#response_max_size ⇒ Integer

Returns Maximum HTTP response size to accept, in bytes.

Returns:

  • (Integer) —

    Maximum HTTP response size to accept, in bytes.



110
111
112
 
# File 'lib/arachni/http/request.rb', line 110

def response_max_size
  @response_max_size
end

#timeout ⇒ Integer

Returns Timeout in milliseconds.

Returns:

  • (Integer) —

    Timeout in milliseconds.



45
46
47
 
# File 'lib/arachni/http/request.rb', line 45

def timeout
  @timeout
end

#username ⇒ String

Returns HTTP username.

Returns:



59
60
61
 
# File 'lib/arachni/http/request.rb', line 59

def username
  @username
end

Class Method Details

.encode(string) ⇒ Object 



790
791
792
793
794
 
# File 'lib/arachni/http/request.rb', line 790

def encode( string )
    string = string.to_s
    @easy ||= Ethon::Easy.new( url: 'www.example.com' )
    ENCODE_CACHE.fetch( string ) { @easy.escape( string ) }
end

.encode_hash(hash, skip = []) ⇒ Object 



770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
 
# File 'lib/arachni/http/request.rb', line 770

def encode_hash( hash, skip = [] )
    hash.inject({}) do |h, (k, v)|

        if skip.include?( k )
            # We need to at least encode null-bytes since they can't
            # be transported at all.
            # If we don't Typhoeus/Ethon will raise errors.
            h.merge!( encode_null_byte( k ) => encode_null_byte( v ) )
        else
            h.merge!( encode( k ) => encode( v ) )
        end

        h
    end
end

.encode_null_byte(string) ⇒ Object 



786
787
788
 
# File 'lib/arachni/http/request.rb', line 786

def encode_null_byte( string )
    string.to_s.gsub "\0", '%00'
end

.from_rpc_data(data) ⇒ Request

Parameters:

Returns:



737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
 
# File 'lib/arachni/http/request.rb', line 737

def from_rpc_data( data )
    instance = allocate
    data.each do |name, value|

        value = case name
                    when 'method', 'mode'
                        value.to_sym

                    else
                        value
                end

        instance.instance_variable_set( "@#{name}", value )
    end
    instance
end

.parse_body(body) ⇒ Hash

Parses an HTTP request body generated by submitting a form.

Parameters:

Returns:

  • (Hash) —

    Parameters.



760
761
762
763
764
765
766
767
768
 
# File 'lib/arachni/http/request.rb', line 760

def parse_body( body )
    return {} if body.to_s.empty?

    body.split( '&' ).inject( {} ) do |h, pair|
        name, value = pair.split( '=', 2 )
        h[Form.decode( name.to_s )] = Form.decode( value )
        h
    end
end

Instance Method Details

#==(other) ⇒ Object 



680
681
682
 
# File 'lib/arachni/http/request.rb', line 680

def ==( other )
    hash == other.hash
end

#asynchronous? ⇒ Boolean

Returns `true` if #mode is `:async`, `false` otherwise.

Returns:

  • (Boolean) —

    `true` if #mode is `:async`, `false` otherwise.



203
204
205
 
# File 'lib/arachni/http/request.rb', line 203

def asynchronous?
    mode == :async
end

#blocking? ⇒ Boolean

Returns `true` if #mode is `:sync`, `false` otherwise.

Returns:

  • (Boolean) —

    `true` if #mode is `:sync`, `false` otherwise.



209
210
211
 
# File 'lib/arachni/http/request.rb', line 209

def blocking?
    mode == :sync
end

#body_parameters ⇒ Object 



270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
 
# File 'lib/arachni/http/request.rb', line 270

def body_parameters
    return {}         if method != :post
    return parameters if parameters.any?

    if headers.content_type.to_s.start_with?( 'multipart/form-data' )
        return {} if !headers.content_type.include?( 'boundary=' )

        return Form.parse_data(
            body,
            headers.content_type.match( /boundary=(.*)/i )[1].to_s
        )
    end

    self.class.parse_body( body )
end

#buffered? ⇒ Boolean

Returns:

  • (Boolean) 


367
368
369
 
# File 'lib/arachni/http/request.rb', line 367

def buffered?
    @on_body.any? || @on_body_line.any? || @on_body_lines.any?
end

#clear_callbacks ⇒ Object

Clears #on_complete callbacks.



339
340
341
342
343
344
345
 
# File 'lib/arachni/http/request.rb', line 339

def clear_callbacks
    @on_complete.clear
    @on_body.clear
    @on_headers.clear
    @on_body_line.clear
    @on_body_lines.clear
end

#effective_cookies ⇒ Object 



244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
 
# File 'lib/arachni/http/request.rb', line 244

def effective_cookies
    effective_cookies = self.cookies.dup

    if !headers['Cookie'].to_s.empty?
        Cookie.from_string( url, headers['Cookie'] ).
            inject( effective_cookies ) do |h, cookie|
            h[cookie.name] ||= cookie.value
            h
        end
    end

    @raw_cookies.inject( effective_cookies ) do |h, cookie|
        h[cookie.raw_name] ||= cookie.raw_value
        h
    end

    effective_cookies
end

#effective_parameters ⇒ Object 



263
264
265
266
267
268
 
# File 'lib/arachni/http/request.rb', line 263

def effective_parameters
    ep = Utilities.uri_parse_query( url )
    return ep if parameters.empty?

    ep.merge!( parameters )
end

#fingerprint? ⇒ Bool

Returns `true` if the Arachni::HTTP::Response should be fingerprinted for platforms, `false` otherwise.

Returns:



356
357
358
 
# File 'lib/arachni/http/request.rb', line 356

def fingerprint?
    @fingerprint
end

#follow_location? ⇒ Bool

Returns `true` if redirects should be followed, `false` otherwise.

Returns:

  • (Bool) —

    `true` if redirects should be followed, `false` otherwise.



349
350
351
 
# File 'lib/arachni/http/request.rb', line 349

def follow_location?
    !!@follow_location
end

#hash ⇒ Object 



684
685
686
 
# File 'lib/arachni/http/request.rb', line 684

def hash
    to_h.hash
end

#high_priority? ⇒ Boolean

Returns:

  • (Boolean) 


174
175
176
 
# File 'lib/arachni/http/request.rb', line 174

def high_priority?
    !!@high_priority
end

#inspect ⇒ Object 



292
293
294
295
296
297
298
299
300
301
302
 
# File 'lib/arachni/http/request.rb', line 292

def inspect
    s = "#<#{self.class} "
    s << "@id=#{id} "
    s << "@mode=#{mode} "
    s << "@method=#{method} "
    s << "@url=#{url.inspect} "
    s << "@parameters=#{parameters.inspect} "
    s << "@high_priority=#{high_priority} "
    s << "@performer=#{performer.inspect}"
    s << '>'
end

#marshal_dump ⇒ Object 



688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
 
# File 'lib/arachni/http/request.rb', line 688

def marshal_dump
    raw_cookies   = @raw_cookies.dup
    callbacks     = @on_complete.dup
    on_body       = @on_body.dup
    on_headers    = @on_headers.dup
    on_body_line  = @on_body_line.dup
    on_body_lines = @on_body_lines.dup
    performer     = @performer
    response      = @response

    @performer     = nil
    @response      = nil
    @raw_cookies   = []
    @on_complete   = []
    @on_body       = []
    @on_body_line  = []
    @on_body_lines = []
    @on_headers    = []

    instance_variables.inject( {} ) do |h, iv|
        next h if iv == :@scope
        h[iv.to_s.gsub('@','')] = instance_variable_get( iv )
        h
    end
ensure
    @response      = response
    @raw_cookies   = raw_cookies
    @on_complete   = callbacks
    @on_body       = on_body
    @on_body_line  = on_body_line
    @on_body_lines = on_body_lines
    @on_headers    = on_headers
    @performer     = performer
end

#marshal_load(h) ⇒ Object 



723
724
725
 
# File 'lib/arachni/http/request.rb', line 723

def marshal_load( h )
    h.each { |k, v| instance_variable_set( "@#{k}", v ) }
end

#method(*args) ⇒ Symbol

Returns HTTP method.

Returns:

  • (Symbol) —

    HTTP method.



215
216
217
218
 
# File 'lib/arachni/http/request.rb', line 215

def method( *args )
    return super( *args ) if args.any? # Preserve Object#method.
    @method
end

#method=(verb) ⇒ Symbol

Note:

Method will be normalized to a lower-case symbol.

Sets the request HTTP method.

Parameters:

  • verb (#to_s) —

    HTTP method.

Returns:

  • (Symbol) —

    HTTP method.



229
230
231
 
# File 'lib/arachni/http/request.rb', line 229

def method=( verb )
    @method = verb.to_s.downcase.to_sym
end

#on_body(&block) ⇒ Object 



320
321
322
323
324
 
# File 'lib/arachni/http/request.rb', line 320

def on_body( &block )
    fail 'Block is missing.' if !block_given?
    @on_body << block
    self
end

#on_body_line(&block) ⇒ Object 



326
327
328
329
330
 
# File 'lib/arachni/http/request.rb', line 326

def on_body_line( &block )
    fail 'Block is missing.' if !block_given?
    @on_body_line << block
    self
end

#on_body_lines(&block) ⇒ Object 



332
333
334
335
336
 
# File 'lib/arachni/http/request.rb', line 332

def on_body_lines( &block )
    fail 'Block is missing.' if !block_given?
    @on_body_lines << block
    self
end

#on_complete(&block) ⇒ Object

Note:

Can be invoked multiple times.

Parameters:

  • block (Block) —

    Callback to be passed the response.



314
315
316
317
318
 
# File 'lib/arachni/http/request.rb', line 314

def on_complete( &block )
    fail 'Block is missing.' if !block_given?
    @on_complete << block
    self
end

#on_headers(&block) ⇒ Object 



304
305
306
307
308
 
# File 'lib/arachni/http/request.rb', line 304

def on_headers( &block )
    fail 'Block is missing.' if !block_given?
    @on_headers << block
    self
end

#prepare_headers ⇒ Object 



797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
 
# File 'lib/arachni/http/request.rb', line 797

def prepare_headers
    headers['User-Agent']      ||= Options.http.user_agent
    headers['Accept']          ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
    headers['From']            ||= Options.authorized_by if Options.authorized_by
    headers['Accept-Language'] ||= 'en-US,en;q=0.8,he;q=0.6'

    headers.each { |k, v| headers[k] = Header.encode( v ) if v }

    final_cookies_hash = self.cookies
    final_raw_cookies  = self.raw_cookies

    if headers['Cookie']
        final_raw_cookies_set = Set.new( final_raw_cookies.map(&:name) )
        final_raw_cookies |= Cookie.from_string( url, headers['Cookie'] ).reject do |c|
            final_cookies_hash.include?( c.name ) ||
                final_raw_cookies_set.include?( c.name )
        end
    end

    headers['Cookie'] = final_cookies_hash.
        map { |k, v| "#{Cookie.encode( k )}=#{Cookie.encode( v )}" }.join( ';' )

    if !headers['Cookie'].empty? && final_raw_cookies.any?
        headers['Cookie'] += ';'
    end

    headers['Cookie'] += final_raw_cookies.map { |c| c.to_s }.join( ';' )

    headers.delete( 'Cookie' ) if headers['Cookie'].empty?

    headers
end

#run ⇒ Response

Note:

Will call #on_complete callbacks.

Performs the Arachni::HTTP::Request without going through Client.

Returns:



394
395
396
 
# File 'lib/arachni/http/request.rb', line 394

def run
    client_run
end

#set_response_data(typhoeus_response) ⇒ Object 



656
657
658
659
660
661
662
663
664
665
666
 
# File 'lib/arachni/http/request.rb', line 656

def set_response_data( typhoeus_response )
    fill_in_data_from_typhoeus_response typhoeus_response

    self.response = Response.from_typhoeus(
        typhoeus_response,
        normalize_url: @normalize_url,
        request:       self
    )

    self.response.update_from_typhoeus typhoeus_response
end

#to_h ⇒ Object 



668
669
670
671
672
673
674
675
676
677
678
 
# File 'lib/arachni/http/request.rb', line 668

def to_h
    {
        url:            url,
        parameters:     parameters,
        headers:        headers,
        headers_string: headers_string,
        effective_body: effective_body,
        body:           body,
        method:         method
    }
end

#to_rpc_data ⇒ Hash

Returns Data representing this instance that are suitable the RPC transmission.

Returns:

  • (Hash) —

    Data representing this instance that are suitable the RPC transmission.



729
730
731
 
# File 'lib/arachni/http/request.rb', line 729

def to_rpc_data
    marshal_dump
end

#to_s ⇒ String

Returns HTTP request string.

Returns:

  • (String) —

    HTTP request string.



288
289
290
 
# File 'lib/arachni/http/request.rb', line 288

def to_s
    "#{headers_string}#{effective_body}"
end

#to_typhoeus ⇒ Typhoeus::Response

Returns `self` converted to a `Typhoeus::Request`.

Returns:

  • (Typhoeus::Response) —

    `self` converted to a `Typhoeus::Request`.



400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
 
# File 'lib/arachni/http/request.rb', line 400

def to_typhoeus
    prepare_headers

    if (userpwd = (@username || Options.http.authentication_username))
        if (passwd = (@password || Options.http.authentication_password))
            userpwd += ":#{passwd}"
        end
    end

    max_size = @response_max_size || Options.http.response_max_size
    # Weird I know, for some reason 0 gets ignored.
    max_size = 1   if max_size == 0
    max_size = nil if max_size < 0

    ep = self.class.encode_hash( self.effective_parameters, @raw_parameters )

    eb = self.body
    if eb.is_a?( Hash )
        eb = self.class.encode_hash( eb, @raw_parameters )
    end

    options = {
        method:          method,
        headers:         headers,

        body:            eb,
        params:          ep,

        userpwd:         userpwd,

        followlocation:  follow_location?,
        maxredirs:       @max_redirects,

        ssl_verifypeer:  !!Options.http.ssl_verify_peer,
        ssl_verifyhost:  Options.http.ssl_verify_host ? 2 : 0,
        sslcert:         Options.http.ssl_certificate_filepath,
        sslcerttype:     Options.http.ssl_certificate_type,
        sslkey:          Options.http.ssl_key_filepath,
        sslkeytype:      Options.http.ssl_key_type,
        sslkeypasswd:    Options.http.ssl_key_password,
        cainfo:          Options.http.ssl_ca_filepath,
        capath:          Options.http.ssl_ca_directory,
        sslversion:      Options.http.ssl_version,

        accept_encoding: 'gzip, deflate',
        nosignal:        true,

        # If Content-Length is missing this option will have no effect, so
        # we'll also stream the body to make sure that we can at least abort
        # the reading of the response body if it exceeds this limit.
        maxfilesize:     max_size,

        # Reusing connections for blocking requests used to cause FD leaks
        # but doesn't appear to do so anymore.
        #
        # Let's allow reuse for all request types again but keep an eye on it.
        # forbid_reuse:    blocking?,

        # Enable debugging messages in order to capture raw traffic data.
        verbose:         true,

        # We're going to be escaping **a lot** of the same strings during
        # the scan, so bypass Ethon's encoding and do our own cache-based
        # encoding.
        escape:          false
    }

    options[:timeout_ms] = timeout if timeout

    # This will allow GSS-Negotiate to work out of the box but shouldn't
    # have any adverse effects.
    if !options[:userpwd] && !parsed_url.user
        options[:userpwd]  = ':'
        options[:httpauth] = :gssnegotiate
    else
        options[:httpauth] = Options.http.authentication_type.to_sym
    end

    if proxy
        options.merge!(
            proxy:     proxy,
            proxytype: (proxy_type || :http).to_sym
        )

        if proxy_user_password
            options[:proxyuserpwd] = proxy_user_password
        end

    elsif Options.http.proxy_host && Options.http.proxy_port
        options.merge!(
            proxy:     "#{Options.http.proxy_host}:#{Options.http.proxy_port}",
            proxytype: (Options.http.proxy_type || :http).to_sym
        )

        if Options.http.proxy_username && Options.http.proxy_password
            options[:proxyuserpwd] =
                "#{Options.http.proxy_username}:#{Options.http.proxy_password}"
        end
    end

    typhoeus_request = Typhoeus::Request.new( url.split( '?').first, options )

    aborted = nil

    # Always set this because we'll be streaming most of the time, so we
    # should set @response so that there'll be a response available for the
    # #on_body and #on_body_line callbacks.
    typhoeus_request.on_headers do |typhoeus_response|
        next aborted if aborted

        set_response_data typhoeus_response

        @on_headers.each do |on_header|
            exception_jail false do
                if on_header.call( self.response ) == :abort
                    break aborted = :abort
                end
            end

            next aborted if aborted
        end
    end

    if @on_body.any?
        typhoeus_request.on_body do |chunk|
            next aborted if aborted

            @on_body.each do |b|
                exception_jail false do
                    chunk.recode!
                    if b.call( chunk, self.response ) == :abort
                        break aborted = :abort
                    end
                end
            end

            next aborted if aborted
        end
    end

    if @on_body_line.any?
        line_buffer = ''
        typhoeus_request.on_body do |chunk|
            next aborted if aborted

            chunk.recode!
            line_buffer << chunk

            lines = line_buffer.lines

            @response_body_buffer = nil

            # Incomplete last line, we've either read everything of were cut
            # short, but we can't know which.
            if !lines.last.index( /[\n\r]/, -1 )
                last_line = lines.pop

                # Set it as the generic body buffer in order to be accessible
                # via #on_complete in case this was indeed the end of the
                # response.
                @response_body_buffer = last_line.dup

                # Also push it back to out own buffer in case there's more
                # to read in order to complete the line.
                line_buffer = last_line
            end

            lines.each do |line|
                @on_body_line.each do |b|
                    exception_jail false do
                        if b.call( line, self.response ) == :abort
                            break aborted = :abort
                        end
                    end
                end

                break aborted if aborted
            end

            line_buffer.clear

            next aborted if aborted
        end
    end

    if @on_body_lines.any?
        lines_buffer = ''
        typhoeus_request.on_body do |chunk|
            next aborted if aborted

            chunk.recode!
            lines_buffer << chunk

            lines, middle, remnant = lines_buffer.rpartition( /[\r\n]/ )
            lines << middle

            @response_body_buffer = nil

            # Incomplete last line, we've either read everything of were cut
            # short, but we can't know which.
            if !remnant.empty?
                # Set it as the generic body buffer in order to be accessible
                # via #on_complete in case this was indeed the end of the
                # response.
                @response_body_buffer = remnant.dup

                # Also push it back to out own buffer in case there's more
                # to read in order to complete the line.
                lines_buffer = remnant
            end

            @on_body_lines.each do |b|
                exception_jail false do
                    if b.call( lines, self.response ) == :abort
                        break aborted = :abort
                    end
                end
            end

            next aborted if aborted
        end
    end

    if @on_complete.any?
        # No need to set our own reader in order to enforce max response size
        # if the response is already been read bit by bit via other callbacks.
        if typhoeus_request.options[:maxfilesize] && @on_body.empty? &&
            @on_body_line.empty? && @on_body_lines.empty?

            @response_body_buffer = ''
            set_body_reader( typhoeus_request, @response_body_buffer )
        end

        typhoeus_request.on_complete do |typhoeus_response|
            next aborted if aborted

            # Set either by the default body reader or is a remnant from
            # a user specified callback like #on_body, #on_body_line, etc.
            if @response_body_buffer
                typhoeus_response.options[:response_body] =
                    @response_body_buffer
            end

            set_response_data typhoeus_response

            @on_complete.each do |b|
                exception_jail false do
                    b.call self.response
                end
            end
        end
    end

    typhoeus_request
end

#train ⇒ Object

Flags that the response should be analyzed by the Trainer for new elements.



373
374
375
 
# File 'lib/arachni/http/request.rb', line 373

def train
    @train = true
end

#train? ⇒ Bool

Returns `true` if the Arachni::HTTP::Response should be analyzed by the Trainer for new elements, `false` otherwise.

Returns:



363
364
365
 
# File 'lib/arachni/http/request.rb', line 363

def train?
    @train
end

#update_cookies ⇒ Object

Flags that the CookieJar should be updated with the Arachni::HTTP::Response cookies.



385
386
387
 
# File 'lib/arachni/http/request.rb', line 385

def update_cookies
    @update_cookies = true
end

#update_cookies? ⇒ Bool

Returns `true` if the CookieJar should be updated with the Arachni::HTTP::Response cookies, `false` otherwise.

Returns:



380
381
382
 
# File 'lib/arachni/http/request.rb', line 380

def update_cookies?
    @update_cookies
end