Class: SpiderInstance

Inherits:
Object
  • Object
show all
Defined in:
lib/spider/spider_instance.rb

Defined Under Namespace

Classes: HeaderSetter

Constant Summary collapse

NOFOLLOW_REL =

Matches an anchor tag's attribute string carrying a rel value that should not be followed (nofollow, sponsored, or ugc), regardless of quoting.

/rel\s*=\s*["']?[^"'>]*\b(?:nofollow|sponsored|ugc)\b/i

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(next_urls, seen = [], rules = nil, robots_seen = []) ⇒ SpiderInstance

:nodoc:



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/spider/spider_instance.rb', line 34

def initialize(next_urls, seen = [], rules = nil, robots_seen = []) # :nodoc:
  @url_checks   = []
  @cache        = :memory
  @callbacks    = {}
  @next_urls    = [ next_urls ]
  @seen         = seen
  @rules        = rules || RobotRules.new("Ruby Spider #{Spider::VERSION}")
  @robots_seen  = robots_seen
  @headers      = {}
  @setup        = nil
  @teardown     = nil
  @interrupted  = false
  @crawl_delay  = 0
  @max_urls     = nil
  @urls_fetched = 0
end

Instance Attribute Details

#crawl_delay=(value) ⇒ Object (writeonly)

Seconds to wait before each HTTP request. Use to be polite to servers. s.crawl_delay = 1

Maximum number of pages to fetch before stopping. nil means no limit. s.max_urls = 100



32
33
34
 
# File 'lib/spider/spider_instance.rb', line 32

def crawl_delay=(value)
  @crawl_delay = value
end

#max_urls=(value) ⇒ Object (writeonly)

Seconds to wait before each HTTP request. Use to be polite to servers. s.crawl_delay = 1

Maximum number of pages to fetch before stopping. nil means no limit. s.max_urls = 100



32
33
34
 
# File 'lib/spider/spider_instance.rb', line 32

def max_urls=(value)
  @max_urls = value
end

Instance Method Details

#add_url_check(&block) ⇒ Object

Add a predicate that determines whether to continue down this URL's path. All predicates must be true in order for a URL to proceed.

Takes a block that takes a string and produces a boolean. For example, this will ensure that the URL starts with 'http://cashcats.biz':

add_url_check { |a_url| a_url =~ %r^http://cashcats.biz.*



58
59
60
 
# File 'lib/spider/spider_instance.rb', line 58

def add_url_check(&block)
  @url_checks << block
end

#allowable_url?(a_url, parsed_url) ⇒ Boolean

:nodoc:

Returns:

  • (Boolean) 


212
213
214
215
 
# File 'lib/spider/spider_instance.rb', line 212

def allowable_url?(a_url, parsed_url) # :nodoc:
  !parsed_url.nil? && !@seen.include?(parsed_url) && allowed?(a_url, parsed_url) &&
    @url_checks.map { |url_check|url_check.call(a_url) }.all?
end

#allowed?(a_url, parsed_url) ⇒ Boolean

True if the robots.txt for that URL allows access to it.

Returns:

  • (Boolean) 


218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
 
# File 'lib/spider/spider_instance.rb', line 218

def allowed?(a_url, parsed_url) # :nodoc:
  return false unless [ "http", "https" ].include?(parsed_url.scheme)
  u = "#{parsed_url.scheme}://#{parsed_url.host}:#{parsed_url.port}/robots.txt"
  parsed_u = URI.parse(u)
  return false unless @url_checks.map { |url_check|url_check.call(a_url) }.all?
  begin
    unless @robots_seen.include?(u)
      # open(u, 'User-Agent' => 'Ruby Spider',
      #  'Accept' => 'text/html,text/xml,application/xml,text/plain', :ssl_verify => false) do |url|
      #  @rules.parse(u, url.read)
      # end
      get_page(parsed_u) do |r|
        @rules.parse(u, r.body)
      end
      @robots_seen << u
    end
    @rules.allowed?(a_url)
  rescue OpenURI::HTTPError
    true # No robots.txt
  rescue StandardError, Timeout::Error # to keep it from crashing
    false
  end
end

#check_already_seen_with(cacher) ⇒ Object

The Web is a graph; to avoid cycles we store the nodes (URLs) already visited. The Web is a really, really, really big graph; as such, this list of visited nodes grows really, really, really big.

Change the object used to store these seen nodes with this. The default object is an instance of Array. Available with Spider is a wrapper of memcached.

You can implement a custom class for this; any object passed to check_already_seen_with must understand just << and included? .

default

check_already_seen_with Array.new

memcached

require 'spider/included_in_memcached' check_already_seen_with IncludedInMemcached.new('localhost:11211')



79
80
81
82
83
84
85
 
# File 'lib/spider/spider_instance.rb', line 79

def check_already_seen_with(cacher)
  if cacher.respond_to?(:<<) && cacher.respond_to?(:include?)
    @seen = cacher
  else
    raise ArgumentError, "expected something that responds to << and included?"
  end
end

#clear_headers ⇒ Object

Reset the headers hash.



170
171
172
 
# File 'lib/spider/spider_instance.rb', line 170

def clear_headers
  @headers = {}
end

#construct_complete_url(base_url, additional_url, parsed_additional_url = nil) ⇒ Object

:nodoc:



308
309
310
311
312
313
 
# File 'lib/spider/spider_instance.rb', line 308

def construct_complete_url(base_url, additional_url, parsed_additional_url = nil) # :nodoc:
  parsed_additional_url ||= URI.parse(additional_url)
  return additional_url unless parsed_additional_url.scheme.nil?
  base = base_url.is_a?(URI) ? base_url : URI.parse(base_url.to_s)
  URI.join(base, additional_url).to_s
end

#do_callbacks(a_url, resp, prior_url) ⇒ Object

:nodoc:



265
266
267
268
269
270
271
272
273
 
# File 'lib/spider/spider_instance.rb', line 265

def do_callbacks(a_url, resp, prior_url) # :nodoc:
  cbs = [ @callbacks[:every],
    resp.success? ?  @callbacks[:success] : @callbacks[:failure],
    @callbacks[resp.code.to_i] ]

  cbs.each do |cb|
    cb.call(a_url, resp, prior_url) if cb
  end
end

#generate_next_urls(a_url, resp) ⇒ Object

:nodoc:



275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
 
# File 'lib/spider/spider_instance.rb', line 275

def generate_next_urls(a_url, resp) # :nodoc:
  # Only scan for links if the content-type is HTML or the URL ends with .html
  content_type = resp["Content-Type"] || resp["content-type"] || ""
  url_ends_with_html = a_url.downcase.end_with?(".html")

  unless content_type.downcase.include?("text/html") || url_ends_with_html
    return []
  end

  web_page = resp.body
  base_url = (web_page.scan(/base\s+href="(.*?)"/i).flatten + [ a_url ])[0]

  # Extract each anchor tag once, then pull href and rel from the same tag,
  # respecting rel="nofollow"/"sponsored"/"ugc".
  web_page.scan(/<a\s([^>]*)>/i).flatten.map do |attrs|
    next nil if attrs =~ NOFOLLOW_REL

    href = attrs[/href\s*=\s*"([^"]*)"/i, 1]
    next nil if href.nil? || href.empty?

    begin
      parsed_link = URI.parse(href)
      if parsed_link.fragment == "#"
        nil
      else
        construct_complete_url(base_url, href, parsed_link)
      end
    rescue StandardError
      nil
    end
  end.compact
end

#get_page(parsed_url, &block) ⇒ Object

:nodoc:



242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
 
# File 'lib/spider/spider_instance.rb', line 242

def get_page(parsed_url, &block) # :nodoc:
  @seen << parsed_url
  begin
    sleep @crawl_delay if @crawl_delay && @crawl_delay > 0
    http = Net::HTTP.new(parsed_url.host, parsed_url.port)
    if parsed_url.scheme == "https"
      http.use_ssl = true
      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    end
    # Uses start because http.finish cannot be called.
    r = http.start { |h| h.request(Net::HTTP::Get.new(parsed_url.request_uri, @headers)) }
    if r.redirect?
      get_page(URI.parse(construct_complete_url(parsed_url, r["Location"])), &block)
    else
      @urls_fetched += 1
      block.call(r)
    end
  rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError => e
    p e
    nil
  end
end

#headers ⇒ Object

Use like a hash: headers = 'user_id=1;password=btrross3'



158
159
160
 
# File 'lib/spider/spider_instance.rb', line 158

def headers
  HeaderSetter.new(self)
end

#on(code, p = nil, &block) ⇒ Object

Add a response handler. A response handler's trigger can be :every, :success, :failure, or any HTTP status code. The handler itself can be either a Proc or a block.

The arguments to the block are: the URL as a string, an instance of Net::HTTPResponse, and the prior URL as a string.

For example:

on 404 do |a_url, resp, prior_url| puts "URL not found: #a_url" end

on :success do |a_url, resp, prior_url| puts a_url puts resp.body end

on :every do |a_url, resp, prior_url| puts "Given this code: #SpiderInstance.respresp.code" end



133
134
135
136
137
138
139
140
141
 
# File 'lib/spider/spider_instance.rb', line 133

def on(code, p = nil, &block)
  f = p ? p : block
  case code
  when Integer
    @callbacks[code] = f
  else
    @callbacks[code.to_sym] = f
  end
end

#raw_headers ⇒ Object

:nodoc:



162
163
164
 
# File 'lib/spider/spider_instance.rb', line 162

def raw_headers # :nodoc:
  @headers
end

#raw_headers=(v) ⇒ Object

:nodoc:



165
166
167
 
# File 'lib/spider/spider_instance.rb', line 165

def raw_headers=(v) # :nodoc:
  @headers = v
end

#setup(p = nil, &block) ⇒ Object

Run before the HTTP request. Given the URL as a string. setup do |a_url| headers = 'user_id=1;admin=true' end



147
148
149
 
# File 'lib/spider/spider_instance.rb', line 147

def setup(p = nil, &block)
  @setup = p ? p : block
end

#start! ⇒ Object

:nodoc:



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/spider/spider_instance.rb', line 174

def start! # :nodoc:
  trap("SIGINT") { @interrupted = true }
  begin
    next_urls = @next_urls.pop
    next_urls.each do |prior_url, urls|
      urls = [ urls ] unless urls.kind_of?(Array)
      urls.map do |a_url|
        [ a_url, (URI.parse(a_url) rescue nil) ]
      end.select do |a_url, parsed_url|
        allowable_url?(a_url, parsed_url)
      end.each do |a_url, parsed_url|
        @setup.call(a_url) unless @setup.nil?
        get_page(parsed_url) do |response|
          do_callbacks(a_url, response, prior_url)
          generate_next_urls(a_url, response).each do |a_next_url|
            @next_urls.push a_url => a_next_url
          end
        end
        @teardown.call(a_url) unless @teardown.nil?
        @interrupted = true if @max_urls && @urls_fetched >= @max_urls
        break if @interrupted
      end
    end
  end while !@next_urls.empty? && !@interrupted
end

#stop! ⇒ Object

:nodoc:



200
201
202
 
# File 'lib/spider/spider_instance.rb', line 200

def stop! # :nodoc:
  @interrupted = true
end

#store_next_urls_with(a_store) ⇒ Object

The Web is a really, really, really big graph; as such, this list of nodes to visit grows really, really, really big.

Change the object used to store nodes we have yet to walk. The default object is an instance of Array. Available with Spider is a wrapper of AmazonSQS.

You can implement a custom class for this; any object passed to check_already_seen_with must understand just push and pop .

default

store_next_urls_with Array.new

AmazonSQS

require 'spider/next_urls_in_sqs' store_next_urls_with NextUrlsInSQS.new(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, queue_name)



103
104
105
106
107
108
109
 
# File 'lib/spider/spider_instance.rb', line 103

def store_next_urls_with(a_store)
  tmp_next_urls = @next_urls
  @next_urls = a_store
  tmp_next_urls.each do |a_url_hash|
    @next_urls.push a_url_hash
  end
end

#success_or_failure(code) ⇒ Object

:nodoc:



204
205
206
207
208
209
210
 
# File 'lib/spider/spider_instance.rb', line 204

def success_or_failure(code) # :nodoc:
  if code > 199 && code < 300
    :success
  else
    :failure
  end
end

#teardown(p = nil, &block) ⇒ Object

Run last, once for each page. Given the URL as a string.



152
153
154
 
# File 'lib/spider/spider_instance.rb', line 152

def teardown(p = nil, &block)
  @teardown = p ? p : block
end