Module: FetchUtil::Regulatory::Headers

Included in:
FetchUtil::Regulatory
Defined in:
lib/fetch_util/regulatory/headers.rb

Instance Method Summary collapse

Instance Method Details

#extract_content_usage_header_signals(headers, path:) ⇒ Object 



12
13
14
15
16
 
# File 'lib/fetch_util/regulatory/headers.rb', line 12

def extract_content_usage_header_signals(headers, path:)
  header_values(headers, "content-usage").flat_map do |value|
    extract_content_usage_statement_signals(value, path: path)
  end
end

#extract_meta_robot_signals(meta_tags, path:) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/fetch_util/regulatory/headers.rb', line 18

def extract_meta_robot_signals(meta_tags, path:)
  signals = []

  meta_tags.each do |attributes|
    if attributes["http-equiv"].to_s.casecmp?("x-robots-tag")
      signals.concat(extract_robot_directive_signals(attributes["content"], path: path))
      next
    end

    name = attributes["name"].to_s.strip
    next if name.empty?
    next if name.casecmp?("tdm-reservation") || name.casecmp?("tdm-policy")
    next unless name.casecmp?("robots") || name.match?(/bot/i)

    signals.concat(extract_robot_directive_signals(attributes["content"], path: path, meta_name: name))
  end

  signals
end

#extract_x_robot_signals(headers, path:) ⇒ Object 



6
7
8
9
10
 
# File 'lib/fetch_util/regulatory/headers.rb', line 6

def extract_x_robot_signals(headers, path:)
  header_values(headers, "x-robots-tag").flat_map do |value|
    extract_robot_directive_signals(value, path: path)
  end
end