Module: PageRequisites

Defined in:
lib/wayback_machine_downloader/page_requisites.rb

Constant Summary collapse

ASSET_REGEX =

regex to find links in href, src, url(), and srcset this ignores data: URIs, mailto:, and anchors

/(?:(href|src|data-src|data-url)\s*=\s*["']([^"']+)["'])|url\(\s*["']?([^"'\)]+)["']?\s*\)|srcset\s*=\s*["']([^"']+)["']/i
PAGE_EXTENSIONS = 
%w[
  .html .htm .shtml .shtm .xhtml
  .asp .aspx .asa .ashx .asmx
  .php .php3 .php4 .php5 .phtml
  .jsp .jspx .do .action
  .cgi .pl .cfm .dll
].freeze

Class Method Summary collapse

Class Method Details

.extract(html_content) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/wayback_machine_downloader/page_requisites.rb', line 13

def self.extract(html_content)
  assets = []

  html_content.scan(ASSET_REGEX) do |match|
    attribute, attr_url, css_url, srcset_url = match
    url = attr_url || css_url || srcset_url
    next unless url

    # href is also used for navigation. Do not turn links to server-rendered
    # pages into prerequisite CDX lookups; doing so can multiply API traffic
    # dramatically on old sites that use extensions such as .php3.
    next if attribute&.downcase == 'href' && page_url?(url)

    # handle srcset (e.g. comma separated values like "image.jpg 1x, image2.jpg 2w")
    if srcset_url
      url.split(',').each do |src_def|
        src_url = src_def.strip.split(' ').first
        assets << src_url if valid_asset?(src_url)
      end
    else
      assets << url if valid_asset?(url)
    end
  end

  assets.uniq
end

.page_url?(url) ⇒ Boolean

Returns:

  • (Boolean) 


40
41
42
43
44
45
46
47
48
49
 
# File 'lib/wayback_machine_downloader/page_requisites.rb', line 40

def self.page_url?(url)
  path = begin
    URI.parse(url).path
  rescue URI::InvalidURIError
    url.to_s.split(/[?#]/, 2).first
  end

  ext = File.extname(path.to_s).downcase
  ext.empty? || PAGE_EXTENSIONS.include?(ext)
end

.valid_asset?(url) ⇒ Boolean

Returns:

  • (Boolean) 


51
52
53
54
55
 
# File 'lib/wayback_machine_downloader/page_requisites.rb', line 51

def self.valid_asset?(url)
  return false if url.strip.empty?
  return false if url.start_with?('data:', 'mailto:', '#', 'javascript:')
  true
end