Class: FetchUtil::Fetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/fetch_util/fetcher.rb

Defined Under Namespace

Classes: PayloadSnapshot

Constant Summary collapse

HOMEPAGE_INDEX_PATTERN = 
Regexp.new(
  "top stories|breaking news|latest news|headlines|" \
  "aktuelle nachrichten|schlagzeilen|neueste nachrichten|" \
  "à la une|dernières nouvelles|actualités|últimas noticias|" \
  "noticias principales|notizie principali|ultime notizie|" \
  "najnowsze wiadomości|najważniejsze|ostatnie wiadomości|aktualności|" \
  "actualiteit|laatste nieuws|senaste nyheter|seneste nyheder|" \
  "siste nytt|tuoreimmat uutiset|aktuálně|legfrissebb|" \
  "cele mai noi știri|aktualności|најновије вести|останні новини|" \
  "τελευταία νέα|güncel haberler|son dakika|senaste nyheterna|" \
  "viktigaste nyheterna|aktualitātes|jaunākās ziņas|naujienos|" \
  "svarbiausios naujienos|главные новости|últimas notícias|" \
  "najnovšie správy|najnovije vijesti|derniers articles",
  Regexp::IGNORECASE
).freeze
DOCS_PORTAL_TITLE_PATTERN = 
/documentation|docs|the ultimate server/i
INDEX_OR_SEARCH_PATH_PATTERN = 
%r{
  /(?:search|s|shop|browse|category|categories|collections?|catalog|keyword|wholesale|
  products?|projects?|jobs?|section|sections|topics?|tags?|archive|archives|latest|headlines|news)/?
}ix
AUTH_PATH_PATTERN = 
%r{
  /(?:log(?:in|-in)|sign(?:in|-in)|auth|oauth|sso|session|sessions|
  accounts?/login|users?/sign_in|password|forgot)(?:/|$)
}ix
ARTICLE_PATH_PATTERN = 
%r{
  /(?:20\d{2}|\d{4}/\d{2}/\d{2}|article|articles|blog|blogs|column|columns|
  entry|entries|post|posts|news/[\w-]+|wiki|dictionary|definition|definitions|thesaurus|\d{5,}[\w-]*\.html?)\b
}ix
LINKED_MARKDOWN_HEADING_PATTERN = 
/(?:^|\s)(?:(?:\d+\.|[-*])\s+)?\#{1,4}\s+\[[^\]]{8,220}\]\(/
LINKED_MARKDOWN_ITEM_PATTERN = 
/(?:^|\s)(?:\d+\.|[-*])\s+\[[^\]]{8,220}\]\(/
INDEX_QUERY_PATTERN = 
/(?:^|[&?])(?:q|query|search|searchtext|keyword|k)=/i
PDF_PATH_PATTERN = 
%r{(?:\.pdf\z|/pdf(?:/|\z)|[?&](?:format|download)=pdf\b)}i
Regexp.new(
  "constitution|constitutional|constitui[cç]?[aã]o|c[oó]digo|codigo|code|statute|act|law|" \
  "regulation|ordinance|decree|treaty|convention",
  Regexp::IGNORECASE
).freeze
/(?:^|\s)(?:Art\.?|Article|Section|Sec\.?|§)\s*(?:\d+[ºª]?|[IVXLCDM]+)/i
/\b(?:title|chapter|part|book|t[ií]tulo|cap[ií]tulo|se[cç][aã]o)\s+(?:[IVXLCDM]+|\d+)/i
Regexp.new(
  "federal republic|rep[úu]blica federativa|republica federativa|civil rights|fundamental rights|" \
  "legal provisions?|official gazette|promulgat|enacted|amended|paragraph|par[áa]grafo|paragrafo|" \
  "inciso|subsection",
  Regexp::IGNORECASE
).freeze
TRACKING_QUERY_PARAM_PATTERNS = 
[
  /\A(?:__goaway_|__cf_chl_)/,
  /\A(?:utm_[a-z]+|fbclid|gclid|lp|mc_cid|mc_eid|ref|source)\z/i,
  /\A__gr(?:sc|ts|ua|rn)\z/
].freeze
TITLE_SLUG_STOPWORDS = 
%w[
  about all and article articles blog book books browse category categories chapter
  collection collections content docs edition editions en for from guide home html
  index latest new news page pages post posts product products search show shop st
  street tag tags the this topic topics unit with work works www your
].freeze
SEARCH_OR_LIST_PATH_SEGMENTS = 
%w[
  archive archives browse catalog categories category collection collections headlines
  jobs keyword latest news product products projects s search section sections shop
  tag tags topic topics wholesale
].freeze
CONTENT_ROUTE_SEGMENTS = 
%w[article articles content paper papers preprint preprints].freeze
SECOND_LEVEL_COUNTRY_TLDS = 
/\A(co|com|org|net|gov|edu|ac)\z/
GOOGLE_HOST_PATTERN = 
/\Agoogle\.[a-z.]+\z/
NETWORK_ERROR_PATTERN = 
Regexp.new(
  "\\b(?:net::ERR_|ERR_NAME_NOT_RESOLVED|pending connections|DNS|resolve|resolution|ENOTFOUND|" \
  "EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT|timed out|timeout|" \
  "connection (?:refused|reset|closed)|disconnected|network)\\b",
  Regexp::IGNORECASE
).freeze
PENDING_CONNECTIONS_FETCH_RETRIES = 
1
PENDING_CONNECTIONS_FETCH_RETRY_WAIT = 
1.0

Instance Method Summary collapse

Constructor Details

#initialize(browser: nil, extractor: nil, **options) ⇒ Fetcher

Returns a new instance of Fetcher.



157
158
159
160
161
162
163
 
# File 'lib/fetch_util/fetcher.rb', line 157

def initialize(browser: nil, extractor: nil, **options)
  @timeout = options.fetch(:timeout, 20)
  @browser = browser || Browser.new(**browser_options(options))
  @extractor = extractor || Extractor.new(reader_mode: options.fetch(:reader_mode, true))
  @raw_docs_fallback = options[:raw_docs_fallback] || RawDocsFallback.new(timeout: @timeout)
  @request_log = options[:request_log]
end

Instance Method Details

#fetch(url) ⇒ Object 



169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
 
# File 'lib/fetch_util/fetcher.rb', line 169

def fetch(url)
  t0 = monotonic_now
  pending_connection_retries = 0

  begin
    result = @browser.with_page(url) do |page|
      payload = @extractor.extract(page)
      build_result(url, page.current_url, payload)
    end
    fallback = seznam_cmp_redirect_fallback_candidate?(url, result) ? @raw_docs_fallback.fetch(url) : nil
    fallback ||= docs_fallback_candidate?(url, result) && poor_docs_result?(result) ? @raw_docs_fallback.fetch(url) : nil
    fallback ||= article_body_fallback_candidate?(result) ? @raw_docs_fallback.fetch(result.final_url) : nil
    result = fallback_result(url, fallback) if fallback
    log_request(url, t0)
    result
  rescue BrowserError, ExtractionError => e
    if e.is_a?(BrowserError) && pending_connections_error?(e) && pending_connection_retries < PENDING_CONNECTIONS_FETCH_RETRIES
      pending_connection_retries += 1
      sleep PENDING_CONNECTIONS_FETCH_RETRY_WAIT
      retry
    end

    fallback = docs_fallback_candidate?(url) ? @raw_docs_fallback.fetch(url) : nil
    if fallback
      result = fallback_result(url, fallback)
      log_request(url, t0)
      return result
    end

    log_request(url, t0)
    return network_error_result(url, e) if e.is_a?(BrowserError) && network_error?(e)

    raise e
  end
end

#quit ⇒ Object 



165
166
167
 
# File 'lib/fetch_util/fetcher.rb', line 165

def quit
  @browser.quit
end