Module: FetchUtil

Defined in:
lib/fetch_util.rb,
lib/fetch_util/cli.rb,
lib/fetch_util/result.rb,
lib/fetch_util/browser.rb,
lib/fetch_util/fetcher.rb,
lib/fetch_util/version.rb,
lib/fetch_util/searcher.rb,
lib/fetch_util/extractor.rb,
lib/fetch_util/regulatory.rb,
lib/fetch_util/request_log.rb,
lib/fetch_util/regulatory/page.rb,
lib/fetch_util/parallel_fetcher.rb,
lib/fetch_util/regulatory/human.rb,
lib/fetch_util/raw_docs_fallback.rb,
lib/fetch_util/regulatory/robots.rb,
lib/fetch_util/browser/navigation.rb,
lib/fetch_util/regulatory/headers.rb,
lib/fetch_util/regulatory/signals.rb,
lib/fetch_util/regulatory/tdm_rep.rb,
lib/fetch_util/regulatory/tdm_page.rb,
lib/fetch_util/regulatory/trust_txt.rb,
lib/fetch_util/browser/stabilization.rb,
lib/fetch_util/regulatory/directives.rb,
lib/fetch_util/regulatory/tdm_policy.rb,
lib/fetch_util/regulatory/cache_store.rb,
lib/fetch_util/regulatory/http_client.rb,
lib/fetch_util/regulatory/robot_globs.rb,
lib/fetch_util/regulatory/tdm_support.rb,
lib/fetch_util/regulatory/fetch_records.rb,
lib/fetch_util/regulatory/orchestration.rb,
lib/fetch_util/searcher/result_filtering.rb,
lib/fetch_util/browser/site_stabilization.rb,
lib/fetch_util/browser/interaction_helpers.rb,
lib/fetch_util/regulatory/source_selection.rb,
lib/fetch_util/regulatory/usage_preferences.rb,
lib/fetch_util/browser/stabilization/page_flow.rb,
lib/fetch_util/browser/navigation/navigator_patch.rb,
lib/fetch_util/browser/stabilization/spa_hydration.rb,
lib/fetch_util/browser/site_stabilization/gitlab_repo.rb,
lib/fetch_util/browser/navigation/headers_and_readiness.rb,
lib/fetch_util/browser/interaction_helpers/timing_helpers.rb,
lib/fetch_util/browser/interaction_helpers/consent_helpers.rb,
lib/fetch_util/browser/interaction_helpers/dom_interaction.rb,
lib/fetch_util/browser/site_stabilization/social_platforms.rb,
lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb

Defined Under Namespace

Classes: Browser, BrowserError, CLI, Error, ExtractionError, Extractor, Fetcher, ParallelFetcher, RawDocsFallback, Regulatory, RequestLog, Result, Searcher

Constant Summary collapse

DOCS_LIKE_EXACT_HOSTS = 
%w[
  developer.mozilla.org
  doc.rust-lang.org
  docs.rs
  fastapi.tiangolo.com
  learn.microsoft.com
  ncbi.nlm.nih.gov
  nextjs.org
  pkg.go.dev
  platform.claude.com
  react.dev
  rubydoc.info
  rubyapi.org
].freeze
DOCS_LIKE_PATH_KEYWORDS = 
%w[
  api
  book
  books
  concept
  concepts
  definition
  definitions
  dictionary
  doc
  docs
  guide
  guides
  howto
  library
  libraries
  manual
  reference
  sdk
  tutorial
].freeze
DOCS_LIKE_PATH_PATTERN = 
%r{/
  (?:
    docs?|reference|api(?:/reference)?|tutorial|guide|guides|library|libraries|
    book|books|dictionary|definition|definitions|concept|concepts|
    get(?:ting)?-started|quick-start|how-to|howto|manual|sdk|learn
  )
  (?:/|\b)
}x
VERSION = 
"0.3.1"

Class Method Summary collapse

Class Method Details

.docs_like_url?(value) ⇒ Boolean

Returns:

  • (Boolean) 


95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/fetch_util.rb', line 95

def docs_like_url?(value)
  uri = value.is_a?(URI::Generic) ? value : URI.parse(value.to_s.strip)
  return false unless uri.is_a?(URI::HTTP) && uri.host

  host = strip_www_host(uri)
  path = uri.path.to_s.downcase
  path_terms = path.split(/[^a-z0-9]+/)

  return true if DOCS_LIKE_EXACT_HOSTS.include?(host)
  return true if host.end_with?(".readthedocs.io")
  return true if host.start_with?("docs.") || host.start_with?("developer.") || host.start_with?("developers.") || host.start_with?("api.")
  return true if host.match?(/\b(?:dictionary|merriam-webster|thefreedictionary|wiktionary|collinsdictionary|reverso)\b/)
  return true if host == "go.dev" && path.match?(%r{\A/ref(?:/|\b)})
  return true if path.match?(DOCS_LIKE_PATH_PATTERN)
  return true if (path_terms & DOCS_LIKE_PATH_KEYWORDS).any?

  false
rescue URI::InvalidURIError
  false
end

.fetch(url, **options) ⇒ Object 



69
70
71
 
# File 'lib/fetch_util.rb', line 69

def fetch(url, **options)
  Fetcher.new(**options).fetch(url)
end

.fetch_many(urls, **options) ⇒ Object 



73
74
75
 
# File 'lib/fetch_util.rb', line 73

def fetch_many(urls, **options)
  ParallelFetcher.new(**options).fetch(urls)
end

.normalize_whitespace(value) ⇒ Object 



85
86
87
88
89
 
# File 'lib/fetch_util.rb', line 85

def normalize_whitespace(value)
  text = value.to_s
  text = text.encode("UTF-8", invalid: :replace, undef: :replace, replace: " ") unless text.encoding == Encoding::UTF_8 && text.valid_encoding?
  text.gsub(/\u00A0/, " ").gsub(/\s+/, " ").strip
end

.regulatory(url, **options) ⇒ Object 



81
82
83
 
# File 'lib/fetch_util.rb', line 81

def regulatory(url, **options)
  Regulatory.new(**options).call(url)
end

.search(query, **options) ⇒ Object 



77
78
79
 
# File 'lib/fetch_util.rb', line 77

def search(query, **options)
  Searcher.new(**options).search(query)
end

.strip_www_host(url) ⇒ Object 



91
92
93
 
# File 'lib/fetch_util.rb', line 91

def strip_www_host(url)
  URI.parse(url.to_s).host.to_s.downcase.sub(/\Awww\./, "")
end