Class: Arachni::HTTP::Client::Dynamic404Handler

Inherits:
Object
  • Object
show all
Includes:
UI::Output, Utilities
Defined in:
lib/arachni/http/client/dynamic_404_handler.rb

Overview

Author:

  • Tasos “Zapotek” Laskos <tasos.laskos@arachni-scanner.com>

Constant Summary collapse

CACHE_SIZE =

Maximum size of the cache that holds 404 signatures.

50
SIGNATURE_THRESHOLD =

Maximum allowed difference ratio when comparing custom 404 signatures. The fact that we refine the signatures allows us to set this threshold really low and still maintain good accuracy.

0.1
PRECISION = 
2

Instance Method Summary collapse

Methods included from Utilities

#available_port, available_port_mutex, #bytes_to_kilobytes, #bytes_to_megabytes, #caller_name, #caller_path, #cookie_decode, #cookie_encode, #cookies_from_file, #cookies_from_parser, #cookies_from_response, #exception_jail, #exclude_path?, #follow_protocol?, #form_decode, #form_encode, #forms_from_parser, #forms_from_response, #full_and_absolute_url?, #generate_token, #get_path, #hms_to_seconds, #html_decode, #html_encode, #include_path?, #links_from_parser, #links_from_response, #normalize_url, #page_from_response, #page_from_url, #parse_set_cookie, #path_in_domain?, #path_too_deep?, #port_available?, #rand_port, #random_seed, #redundant_path?, #regexp_array_match, #remove_constants, #request_parse_body, #seconds_to_hms, #skip_page?, #skip_path?, #skip_resource?, #skip_response?, #to_absolute, #uri_decode, #uri_encode, #uri_parse, #uri_parse_query, #uri_parser, #uri_rewrite

Methods included from UI::Output

#debug?, #debug_level_1?, #debug_level_2?, #debug_level_3?, #debug_level_4?, #debug_off, #debug_on, #disable_only_positives, #included, #mute, #muted?, #only_positives, #only_positives?, #print_bad, #print_debug, #print_debug_backtrace, #print_debug_level_1, #print_debug_level_2, #print_debug_level_3, #print_debug_level_4, #print_error, #print_error_backtrace, #print_exception, #print_info, #print_line, #print_ok, #print_status, #print_verbose, #reroute_to_file, #reroute_to_file?, reset_output_options, #unmute, #verbose?, #verbose_on

Constructor Details

#initialize ⇒ Dynamic404Handler

Returns a new instance of Dynamic404Handler.



28
29
30
31
 
# File 'lib/arachni/http/client/dynamic_404_handler.rb', line 28

def initialize
    @static     = Support::LookUp::HashSet.new
    @signatures = Hash.new
end

Instance Method Details

#_404?(response, &block) ⇒ Boolean

Parameters:

  • response (Response) —

    Checks whether or not the provided response means 'not found'.

  • block (Block) —

    To be passed `true` or `false` depending on the result of the analysis.

Returns:

  • (Boolean) 


37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/arachni/http/client/dynamic_404_handler.rb', line 37

def _404?( response, &block )
    # This matters, the request URL may differ from the response one due to
    # redirections and we need to test the original.
    url = response.request.url

    # Easy pickins, well-behaved static 404 handler and a URL that doesn't
    # need advanced analysis.
    if checked_and_static?( url )
        result = (response.code == 404)
        print_debug "[static]: #{block} #{url} #{result}"
        block.call( result )
        return
    end

    # We've hit the cache, hopefully some preliminary signature will
    # match the response body and we'll get to avoid the advanced analysis.
    if checked?( url )
        result = matches_preliminary_signatures?( url, response.body )

        # If we've got a positive result that's all we need to know, return
        # it immediately.
        if result
            print_debug "[cached]: #{block} #{url} #{result}"
            return block.call( result )
        end

        # If the result was negative only return it if there's no need for
        # advanced analysis for this resource.
        if !needs_advanced_analysis?( url )
            print_debug "[cached]: #{block} #{url} #{result}"
            return block.call( result )
        end
    end

    # No need to go over this process for each caller for the same handler,
    # group them together and they'll get notified when the analysis is
    # complete.
    data_for( url )[:waiting] << [url, response.code, response.body, block]
    if data_for( url )[:in_progress]
        print_debug "[waiting]: #{url} #{block}"
        return
    end
    data_for( url )[:in_progress] = true

    # If it's already checked then preliminary analysis has been performed
    # and since its results can be shared across different resource checks
    # don't waste time redoing it, we can jump straight into the advanced
    # analysis.
    if checked?( url ) && needs_advanced_analysis?( url )
        print_debug "[checking-advanced]: #{url} #{block}"
        process_advanced_analysis_callers_for( url )
        return
    end

    print_debug "[checking]: #{url} #{block}"

    # So... we've got nothing cached for the handler for this URL, let's
    # start from scratch.
    preliminary_analysis( url ) do
        process_callers_for( url )
    end

    nil
end

#checked?(url) ⇒ Bool

Returns `true` if the `url` has been checked for the existence of a custom-404 handler, `false` otherwise.

Parameters:

  • url (String) —

    URL to check.

Returns:

  • (Bool) —

    `true` if the `url` has been checked for the existence of a custom-404 handler, `false` otherwise.



108
109
110
 
# File 'lib/arachni/http/client/dynamic_404_handler.rb', line 108

def checked?( url )
    data_for( url )[:analyzed]
end

#checked_and_static?(url) ⇒ Bool

Returns `true` if the `url` has been checked for the existence of a custom-404 handler but none was identified, `false` otherwise.

Parameters:

  • url (String) —

    URL to check.

Returns:

  • (Bool) —

    `true` if the `url` has been checked for the existence of a custom-404 handler but none was identified, `false` otherwise.



118
119
120
 
# File 'lib/arachni/http/client/dynamic_404_handler.rb', line 118

def checked_and_static?( url )
    @static.include?( url_for( url ) ) && !needs_advanced_analysis?( url )
end

#needs_check?(url) ⇒ Bool

Returns `true` if the `url` needs to be checked for a #_404?, `false` otherwise.

Parameters:

  • url (String) —

    URL to check.

Returns:

  • (Bool) —

    `true` if the `url` needs to be checked for a #_404?, `false` otherwise.



128
129
130
 
# File 'lib/arachni/http/client/dynamic_404_handler.rb', line 128

def needs_check?( url )
    !checked?( url ) || !checked_and_static?( url )
end

#prune ⇒ Object 



138
139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'lib/arachni/http/client/dynamic_404_handler.rb', line 138

def prune
    return if @signatures.size <= CACHE_SIZE

    @signatures.keys.each do |url|
        # If the path hasn't been analyzed yet skip it.
        next if !@signatures[url][:analyzed]

        # We've done enough...
        return if @signatures.size <= CACHE_SIZE

        @signatures.delete( url )
    end
end

#signatures ⇒ Object 



133
134
135
 
# File 'lib/arachni/http/client/dynamic_404_handler.rb', line 133

def signatures
    @signatures
end