Class: FetchUtil::Fetcher

Inherits:

Object

Object
FetchUtil::Fetcher

show all

Defined in:: lib/fetch_util/fetcher.rb

Constant Summary collapse

HOMEPAGE_INDEX_PATTERN =

Regexp.new(
  "top stories|breaking news|latest news|headlines|" \
  "aktuelle nachrichten|schlagzeilen|neueste nachrichten|" \
  "à la une|dernières nouvelles|actualités|últimas noticias|" \
  "noticias principales|notizie principali|ultime notizie|" \
  "najnowsze wiadomości|najważniejsze|ostatnie wiadomości|aktualności|" \
  "actualiteit|laatste nieuws|senaste nyheter|seneste nyheder|" \
  "siste nytt|tuoreimmat uutiset|aktuálně|legfrissebb|" \
  "cele mai noi știri|aktualności|најновије вести|останні новини|" \
  "τελευταία νέα|güncel haberler|son dakika|senaste nyheterna|" \
  "viktigaste nyheterna|aktualitātes|jaunākās ziņas|naujienos|" \
  "svarbiausios naujienos|главные новости|últimas notícias|" \
  "najnovšie správy|najnovije vijesti|derniers articles",
  Regexp::IGNORECASE
).freeze

DOCS_PORTAL_TITLE_PATTERN =

/documentation|docs|the ultimate server/i

INDEX_OR_SEARCH_PATH_PATTERN =

%r{
  /(?:search|s|shop|browse|category|categories|collections?|catalog|keyword|wholesale|
  products?|projects?|jobs?|section|sections|topics?|tags?|archive|archives|latest|headlines|news)/?
}ix

AUTH_PATH_PATTERN =

%r{
  /(?:log(?:in|-in)|sign(?:in|-in)|auth|oauth|sso|session|sessions|
  accounts?/login|users?/sign_in|password|forgot)(?:/|$)
}ix

ARTICLE_PATH_PATTERN =

%r{
  /(?:20\d{2}|\d{4}/\d{2}/\d{2}|article|articles|blog|blogs|column|columns|
  entry|entries|post|posts|news/[\w-]+|wiki|dictionary|definition|definitions|thesaurus|\d{5,}[\w-]*\.html?)\b
}ix

LINKED_MARKDOWN_HEADING_PATTERN =

/(?:^|\s)(?:(?:\d+\.|[-*])\s+)?\#{1,4}\s+\[[^\]]{8,220}\]\(/

LINKED_MARKDOWN_ITEM_PATTERN =

/(?:^|\s)(?:\d+\.|[-*])\s+\[[^\]]{8,220}\]\(/

INDEX_QUERY_PATTERN =

/(?:^|[&?])(?:q|query|search|searchtext|keyword|k)=/i

PDF_PATH_PATTERN =

%r{(?:\.pdf\z|/pdf(?:/|\z)|[?&](?:format|download)=pdf\b)}i

LEGAL_STATUTE_TITLE_PATTERN =

Regexp.new(
  "constitution|constitutional|constitui[cç]?[aã]o|c[oó]digo|codigo|code|statute|act|law|" \
  "regulation|ordinance|decree|treaty|convention",
  Regexp::IGNORECASE
).freeze

LEGAL_PROVISION_MARKER_PATTERN =

/(?:^|\s)(?:Art\.?|Article|Section|Sec\.?|§)\s*(?:\d+[ºª]?|[IVXLCDM]+)/i

LEGAL_PROVISION_STRUCTURAL_PATTERN =

/\b(?:title|chapter|part|book|t[ií]tulo|cap[ií]tulo|se[cç][aã]o)\s+(?:[IVXLCDM]+|\d+)/i

LEGAL_PROVISION_TERMS_PATTERN =

Regexp.new(
  "federal republic|rep[úu]blica federativa|republica federativa|civil rights|fundamental rights|" \
  "legal provisions?|official gazette|promulgat|enacted|amended|paragraph|par[áa]grafo|paragrafo|" \
  "inciso|subsection",
  Regexp::IGNORECASE
).freeze

STRIPPED_QUERY_PARAM_PATTERNS =

[
  /\A(?:__goaway_|__cf_chl_)/,
  /\A(?:utm_[a-z]+|fbclid|gclid|mc_cid|mc_eid)\z/,
  /\A__gr(?:sc|ts|ua|rn)\z/
].freeze

TITLE_SLUG_STOPWORDS =

%w[
  about all and article articles blog book books browse category categories chapter
  collection collections content docs edition editions en for from guide home html
  index latest new news page pages post posts product products search show shop st
  street tag tags the this topic topics unit with work works www your
].freeze

SEARCH_OR_LIST_PATH_SEGMENTS =

%w[
  archive archives browse catalog categories category collection collections headlines
  jobs keyword latest news product products projects s search section sections shop
  tag tags topic topics wholesale
].freeze

CONTENT_ROUTE_SEGMENTS =

%w[article articles content paper papers preprint preprints].freeze

SECOND_LEVEL_COUNTRY_TLDS =

/\A(co|com|org|net|gov|edu|ac)\z/

GOOGLE_HOST_PATTERN =

/\Agoogle\.[a-z.]+\z/

NETWORK_ERROR_PATTERN =

Regexp.new(
  "\\b(?:net::ERR_|ERR_NAME_NOT_RESOLVED|DNS|resolve|resolution|ENOTFOUND|" \
  "EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT|timed out|timeout|" \
  "connection (?:refused|reset|closed)|disconnected|network)\\b",
  Regexp::IGNORECASE
).freeze

Instance Method Summary collapse

#fetch(url) ⇒ Object
#initialize(browser: nil, extractor: nil, **options) ⇒ Fetcher constructor
A new instance of Fetcher.
#quit ⇒ Object

Constructor Details

#initialize(browser: nil, extractor: nil, **options) ⇒ `Fetcher`

Returns a new instance of Fetcher.

# File 'lib/fetch_util/fetcher.rb', line 78

def initialize(browser: nil, extractor: nil, **options)
  @timeout = options.fetch(:timeout, 20)
  @browser = browser || Browser.new(**browser_options(options))
  @extractor = extractor || Extractor.new(reader_mode: options.fetch(:reader_mode, true))
  @raw_docs_fallback = options[:raw_docs_fallback] || RawDocsFallback.new(timeout: @timeout)
  @request_log = options[:request_log]
end

Instance Method Details

#fetch(url) ⇒ `Object`

# File 'lib/fetch_util/fetcher.rb', line 90

def fetch(url)
  t0 = monotonic_now
  result = @browser.with_page(url) do |page|
    payload = @extractor.extract(page)
    build_result(url, page.current_url, payload)
  end
  fallback = docs_fallback_candidate?(url, result) && poor_docs_result?(result) ? @raw_docs_fallback.fetch(url) : nil
  fallback ||= article_body_fallback_candidate?(result) ? @raw_docs_fallback.fetch(result.final_url) : nil
  result = fallback_result(url, fallback) if fallback
  log_request(url, t0)
  result
rescue BrowserError, ExtractionError => e
  fallback = docs_fallback_candidate?(url) ? @raw_docs_fallback.fetch(url) : nil
  if fallback
    result = fallback_result(url, fallback)
    log_request(url, t0)
    return result
  end

  log_request(url, t0)
  return network_error_result(url, e) if e.is_a?(BrowserError) && network_error?(e)

  raise e
end

#quit ⇒ `Object`



86
87
88

# File 'lib/fetch_util/fetcher.rb', line 86

def quit
  @browser.quit
end