Class: FetchUtil::Browser

Inherits:
Object
  • Object
show all
Includes:
InteractionHelpers, Navigation, SiteStabilization, Stabilization
Defined in:
lib/fetch_util/browser.rb,
lib/fetch_util/browser/navigation.rb,
lib/fetch_util/browser/stabilization.rb,
lib/fetch_util/browser/site_stabilization.rb,
lib/fetch_util/browser/interaction_helpers.rb,
lib/fetch_util/browser/stabilization/page_flow.rb,
lib/fetch_util/browser/navigation/navigator_patch.rb,
lib/fetch_util/browser/stabilization/spa_hydration.rb,
lib/fetch_util/browser/site_stabilization/gitlab_repo.rb,
lib/fetch_util/browser/navigation/headers_and_readiness.rb,
lib/fetch_util/browser/interaction_helpers/timing_helpers.rb,
lib/fetch_util/browser/interaction_helpers/consent_helpers.rb,
lib/fetch_util/browser/interaction_helpers/dom_interaction.rb,
lib/fetch_util/browser/site_stabilization/social_platforms.rb,
lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb

Defined Under Namespace

Modules: InteractionHelpers, Navigation, SiteStabilization, Stabilization

Constant Summary collapse

BROWSER_CANDIDATES =

Prefer full Chromium over headless_shell: the full browser stays closer to standard Chromium behavior and exposes APIs that some sites expect, which improves extraction consistency. headless_shell diverges more often and can change page behavior in ways that degrade extraction.

[
  "/usr/bin/chromium-browser",
  "/usr/bin/chromium",
  "/usr/bin/google-chrome",
  "/usr/lib64/chromium-browser/headless_shell"
].freeze
DEFAULT_VIEWPORT = 
{ width: 1366, height: 900 }.freeze
DEFAULT_USER_AGENT = 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " \
"(KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
DEFAULT_ACCEPT_LANGUAGE = 
"en-US,en;q=0.9"
SOCIAL_LOGIN_PHASE_WAIT = 
0.3
POST_CONSENT_IDLE_TIMEOUT = 
3.0
CONTENT_READY_MIN_LENGTH = 
200
SPA_HYDRATION_TIMEOUT = 
2.0
SPA_HYDRATION_POLL = 
0.15
2

Instance Method Summary collapse

Constructor Details

#initialize(timeout: 20, wait: 0.75, wait_for_idle: true, idle_duration: 0.35, viewport: DEFAULT_VIEWPORT, user_agent: DEFAULT_USER_AGENT, accept_language: DEFAULT_ACCEPT_LANGUAGE, browser_path: nil, browser_options: nil) ⇒ Browser

Returns a new instance of Browser.



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/fetch_util/browser.rb', line 41

def initialize(timeout: 20, wait: 0.75, wait_for_idle: true, idle_duration: 0.35,
               viewport: DEFAULT_VIEWPORT, user_agent: DEFAULT_USER_AGENT,
               accept_language: DEFAULT_ACCEPT_LANGUAGE, browser_path: nil,
               browser_options: nil)
  @timeout = timeout.to_f
  @wait = wait.to_f
  @wait_for_idle = wait_for_idle
  @idle_duration = idle_duration.to_f
  @viewport = DEFAULT_VIEWPORT.merge(symbolize_hash(viewport || {}))
  @user_agent = user_agent
  @accept_language = accept_language
  @browser_path = browser_path || ENV["BROWSER_PATH"] || BROWSER_CANDIDATES.find { |path| File.executable?(path) }
  @full_browser = @browser_path && !@browser_path.include?("headless_shell")
  default_opts = { "no-sandbox": nil }
  # Use newer headless mode with the full browser binary for closer runtime
  # parity with standard Chromium. Also disable Ferrum's
  # `enable-automation` flag to reduce tool-specific browser-state
  # differences during extraction.
  if @full_browser
    default_opts["headless"] = "new"
    default_opts["enable-automation"] = false # override Ferrum default
  end
  @browser_options = default_opts.merge(browser_options || {})
  @ferrum = nil
  @mutex = Mutex.new
end

Instance Method Details

#quit ⇒ Object

Shut down the underlying Chromium process. Safe to call multiple times or when no browser has been started yet. After quit, a subsequent with_page call will transparently launch a new process.



101
102
103
104
105
106
 
# File 'lib/fetch_util/browser.rb', line 101

def quit
  @mutex.synchronize do
    @ferrum&.quit
    @ferrum = nil
  end
end

#with_page(url) ⇒ Object

Navigate to url in a fresh browser tab, stabilize the page, then yield the Ferrum::Page to the caller. The page is closed after the block returns (or on error), but the underlying Chromium process is kept alive for reuse by subsequent calls.



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/fetch_util/browser.rb', line 72

def with_page(url)
  raise BrowserError, "No Chromium browser found. Set BROWSER_PATH or install Chromium." unless @browser_path

  ferrum = ensure_browser
  page = ferrum.create_page
  page.headers.set(default_headers)
  page.bypass_csp
  retries = 0
  begin
    page.go_to(url)
  rescue Ferrum::PendingConnectionsError, Ferrum::TimeoutError
    unless page_loaded_enough?(page)
      raise if retries >= NAVIGATION_MAX_RETRIES

      retries += 1
      retry
    end
  end
  stabilize_page(page, url)
  yield page
rescue Ferrum::Error => e
  raise BrowserError, e.message
ensure
  page&.close
end