Module: URLRewrite

Included in:
WaybackMachineDownloader
Defined in:
lib/wayback_machine_downloader/url_rewrite.rb

Constant Summary collapse

SERVER_SIDE_EXTS =

server-side extensions that should work locally

%w[
  .php .php3 .phtml 
  .asp .aspx .ashx .asmx .asa 
  .jsp .jspx .do .action 
  .cgi .pl .py .cfm .shtml
].freeze

Instance Method Summary collapse

Instance Method Details

#normalize_path_for_local(path, root_prefix = './') ⇒ Object 



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/wayback_machine_downloader/url_rewrite.rb', line 79

def normalize_path_for_local(path, root_prefix = './')
  path = path.to_s.strip
  return "#{root_prefix}index.html" if path.empty? || path == "/"

  path_part, query_part = path.split('?', 2)
  path_part = "/index.html" if path_part.nil? || path_part.empty? || path_part == "/"

  # hash query parameters to match sanitize_and_prepare_id
  if query_part && !query_part.empty?
    q_digest = Digest::SHA256.hexdigest(query_part)[0, 12]
    if path_part.include?('.')
      pre, _sep, post = path_part.rpartition('.')
      path_part = "#{pre}__q#{q_digest}.#{post}"
    else
      path_part = "#{path_part}__q#{q_digest}"
    end
  end

  # check if this is a server-side script
  ext = File.extname(path_part).downcase
  unless SERVER_SIDE_EXTS.include?(ext)
    if path_part.end_with?('/') || !path_part.include?('.')
      path_part = "#{path_part.chomp('/')}/index.html"
    end
  end

  clean_path = path_part.sub(%r{\A/+}, '')
  "#{root_prefix}#{clean_path}"
end

#rewrite_css_urls(content, root_prefix = './') ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/wayback_machine_downloader/url_rewrite.rb', line 41

def rewrite_css_urls(content, root_prefix = './')
  target_host = current_host

  # rewrite URLs in CSS
  content = content.gsub(/url\(\s*["']?https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"'\)]+(\/[^"'\)]*)?["']?\s*\)/i) do
    path = $1 || "/index.html"
    local = normalize_path_for_local(path, root_prefix)
    "url(\"#{local}\")"
  end

  # rewrite absolute URLs in CSS
  if target_host && !target_host.empty?
    content = content.gsub(/url\(\s*["']?https?:\/\/#{Regexp.escape(target_host)}(?::\d+)?(\/[^"'\)]*)?["']?\s*\)/i) do
      path = $1 || "/index.html"
      local = normalize_path_for_local(path, root_prefix)
      "url(\"#{local}\")"
    end
  end

  # rewrite root-relative in CSS
  content = content.gsub(/url\(\s*["']?\/([^"'\)\/][^"'\)]*?)["']?\s*\)/i) do
    path = $1
    local = normalize_path_for_local("/#{path}", root_prefix)
    "url(\"#{local}\")"
  end

  content
end

#rewrite_html_attr_urls(content, root_prefix = './') ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/wayback_machine_downloader/url_rewrite.rb', line 12

def rewrite_html_attr_urls(content, root_prefix = './')
  target_host = current_host

  # rewrite URLs to relative paths
  content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"']+(\/[^"']*)?(["'])/i) do
    prefix, path, suffix = $1, ($2 || "/index.html"), $3
    local = normalize_path_for_local(path, root_prefix)
    "#{prefix}#{local}#{suffix}"
  end

  # rewrite absolute URLs to same domain as relative
  if target_host && !target_host.empty?
    content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/#{Regexp.escape(target_host)}(?::\d+)?(\/[^"']*)?(["'])/i) do
      prefix, path, suffix = $1, ($2 || "/index.html"), $3
      local = normalize_path_for_local(path, root_prefix)
      "#{prefix}#{local}#{suffix}"
    end
  end

  # rewrite root-relative URLs
  content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])\/([^"'\/][^"']*)(["'])/i) do
    prefix, path, suffix = $1, $2, $3
    local = normalize_path_for_local("/#{path}", root_prefix)
    "#{prefix}#{local}#{suffix}"
  end

  content
end

#rewrite_js_urls(content, root_prefix = './') ⇒ Object 



70
71
72
73
74
75
76
77
 
# File 'lib/wayback_machine_downloader/url_rewrite.rb', line 70

def rewrite_js_urls(content, root_prefix = './')
  # rewrite archive.org URLs in JavaScript strings
  content.gsub(/(["'])https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"']+(\/[^"']*)?(["'])/i) do
    quote_start, path, quote_end = $1, ($2 || "/index.html"), $3
    local = normalize_path_for_local(path, root_prefix)
    "#{quote_start}#{local}#{quote_end}"
  end
end