Class: FetchUtil::RawDocsFallback

Inherits:
Object
  • Object
show all
Defined in:
lib/fetch_util/raw_docs_fallback.rb

Constant Summary collapse

DEFAULT_HEADERS = 
{
  "User-Agent" => Browser::DEFAULT_USER_AGENT,
  "Accept-Language" => Browser::DEFAULT_ACCEPT_LANGUAGE
}.freeze
BLOCK_ELEMENTS = 
%w[h1 h2 h3 h4 h5 h6 p pre ul ol li table tr].freeze
BLOCK_SELECTOR = 
BLOCK_ELEMENTS.join(", ").freeze
DROP_SELECTORS = 
[
  "script",
  "style",
  "nav",
  "aside",
  "footer",
  ".discord",
  ".toc",
  ".sidebar",
  ".breadcrumbs",
  "[aria-label*='breadcrumb']",
  ".headerlink",
  ".copybutton",
  "button"
].freeze
DOCS_ROOT_SELECTORS = 
[
  "main .sl-markdown-content",
  "main [data-pagefind-body]",
  ".sl-markdown-content",
  "[data-pagefind-body]",
  "main article",
  "main",
  "article",
  "[role='main']",
  "[class*='content']",
  "[id*='content']",
  "[class*='article']",
  "[id*='article']",
  ".content",
  ".resource-container"
].freeze
PRUNED_TEXT_PATTERN = 
/\A(?:on this page|table of contents|edit this page|copy page|copy item path|search|settings|help|expand description)\z/i

Instance Method Summary collapse

Constructor Details

#initialize(timeout: 20) ⇒ RawDocsFallback

Returns a new instance of RawDocsFallback.



50
51
52
 
# File 'lib/fetch_util/raw_docs_fallback.rb', line 50

def initialize(timeout: 20)
  @timeout = timeout.to_i
end

Instance Method Details

#fetch(url) ⇒ Object 



54
55
56
57
58
59
60
61
62
 
# File 'lib/fetch_util/raw_docs_fallback.rb', line 54

def fetch(url)
  final_url, html = fetch_html(url)
  payload = payload_from_html(html, requested_url: url, final_url: final_url)
  return nil unless payload

  [final_url, payload]
rescue Error, SocketError, SystemCallError, Timeout::Error, URI::InvalidURIError
  nil
end

#payload_from_html(html, requested_url:, final_url: requested_url) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/fetch_util/raw_docs_fallback.rb', line 64

def payload_from_html(html, requested_url:, final_url: requested_url)
  document = Nokogiri::HTML(html)
  root = fragment_root(document, final_url) || docs_root(document)
  return nil unless root

  prune!(root)
  title = [fragment_title(document, final_url), first_heading(root), meta_title(document), document.title]
          .map { |candidate| clean_text(candidate) }
          .find { |candidate| !candidate.empty? }
  markdown = markdown_from_root(root, title)
  return nil if clean_text(markdown).length < 40

  {
    "title" => title,
    "byline" => meta_value(document, "author"),
    "excerpt" => first_paragraph(root),
    "siteName" => meta_value(document, "og:site_name", attr: "property") || safe_host(final_url),
    "publishedTime" => meta_value(document, "article:published_time", attr: "property") || meta_value(document, "publish-date"),
    "canonicalUrl" => canonical_url(document, final_url),
    "language" => document.at_css("html")&.[]("lang") || "en",
    "html" => root.to_html,
    "markdown" => markdown,
    "readerMode" => false,
    "contentType" => "article",
    "suspect" => false,
    "warnings" => []
  }
end