Class: FetchUtil::Regulatory

Inherits:
Object
  • Object
show all
Includes:
CacheStore, Directives, FetchRecords, Headers, Human, Orchestration, Page, RobotGlobs, Robots, Signals, SourceSelection, TdmPage, TdmPolicy, TdmRep, TdmSupport, TrustTxt, UsagePreferences
Defined in:
lib/fetch_util/regulatory.rb,
lib/fetch_util/regulatory/page.rb,
lib/fetch_util/regulatory/human.rb,
lib/fetch_util/regulatory/robots.rb,
lib/fetch_util/regulatory/headers.rb,
lib/fetch_util/regulatory/signals.rb,
lib/fetch_util/regulatory/tdm_rep.rb,
lib/fetch_util/regulatory/tdm_page.rb,
lib/fetch_util/regulatory/trust_txt.rb,
lib/fetch_util/regulatory/directives.rb,
lib/fetch_util/regulatory/tdm_policy.rb,
lib/fetch_util/regulatory/cache_store.rb,
lib/fetch_util/regulatory/http_client.rb,
lib/fetch_util/regulatory/robot_globs.rb,
lib/fetch_util/regulatory/tdm_support.rb,
lib/fetch_util/regulatory/fetch_records.rb,
lib/fetch_util/regulatory/orchestration.rb,
lib/fetch_util/regulatory/source_selection.rb,
lib/fetch_util/regulatory/usage_preferences.rb

Defined Under Namespace

Modules: CacheStore, Directives, FetchRecords, Headers, Human, Orchestration, Page, RobotGlobs, Robots, Signals, SourceSelection, TdmPage, TdmPolicy, TdmRep, TdmSupport, TrustTxt, UsagePreferences Classes: HttpClient, Response

Constant Summary collapse

CACHE_TTL = 
86_400
CACHE_VERSION = 
2
DEFAULT_CACHE_PATH = 
File.expand_path("~/.local/state/fetch_util/regulatory-cache")
MACHINE_SOURCES = 
%w[
  robotstxt
  contentsignal
  contentusagerobots
  contentusageheader
  trusttxt
  xrobotstag
  metarobots
  tdmrep
  tdmheaders
  tdmmeta
  tdmpolicy
].freeze
HUMAN_SOURCES = 
%w[human].freeze
SOURCE_CLASSES = 
{
  "machine" => MACHINE_SOURCES,
  "human" => HUMAN_SOURCES
}.freeze

Constants included from Human

Human::HUMAN_PATTERNS

Constants included from Robots

Robots::ROBOT_DIRECTIVES

Method Summary

Methods included from Human

#extract_human_signals

Methods included from TdmPolicy

#expanded_tdm_policy_signals, #extract_tdm_policy_signals, #tdm_policy_record

Methods included from TdmRep

#extract_tdmrep_signals, #tdmrep_record

Methods included from Page

#empty_page_record, #page_record

Methods included from UsagePreferences

#extract_content_signal_signals, #extract_content_usage_robot_signals, #extract_content_usage_statement_signals, #sort_usage_preference_signals

Methods included from TrustTxt

#extract_trusttxt_signals, #trusttxt_record

Methods included from TdmPage

#extract_tdm_meta_signals, #extract_tdm_value_signals

Methods included from Headers

#extract_content_usage_header_signals, #extract_meta_robot_signals, #extract_x_robot_signals

Methods included from Robots

#extract_robots_source_signals, #robots_record

Methods included from Orchestration

#call, #initialize