Class: FetchUtil::Browser

Inherits:
Object
  • Object
show all
Includes:
InteractionHelpers, Navigation, SiteStabilization, Stabilization
Defined in:
lib/fetch_util/browser.rb,
lib/fetch_util/browser/navigation.rb,
lib/fetch_util/browser/stabilization.rb,
lib/fetch_util/browser/site_stabilization.rb,
lib/fetch_util/browser/interaction_helpers.rb,
lib/fetch_util/browser/stabilization/page_flow.rb,
lib/fetch_util/browser/navigation/navigator_patch.rb,
lib/fetch_util/browser/stabilization/spa_hydration.rb,
lib/fetch_util/browser/site_stabilization/gitlab_repo.rb,
lib/fetch_util/browser/navigation/headers_and_readiness.rb,
lib/fetch_util/browser/interaction_helpers/timing_helpers.rb,
lib/fetch_util/browser/interaction_helpers/consent_helpers.rb,
lib/fetch_util/browser/interaction_helpers/dom_interaction.rb,
lib/fetch_util/browser/site_stabilization/social_platforms.rb,
lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb

Defined Under Namespace

Modules: InteractionHelpers, Navigation, SiteStabilization, Stabilization

Constant Summary collapse

BROWSER_CANDIDATES =

Prefer full Chromium over headless_shell: the full browser stays closer to standard Chromium behavior and exposes APIs that some sites expect, which improves extraction consistency. headless_shell diverges more often and can change page behavior in ways that degrade extraction.

[
  "/usr/bin/chromium-browser",
  "/usr/bin/chromium",
  "/usr/bin/google-chrome",
  "/usr/lib64/chromium-browser/headless_shell"
].freeze
DEFAULT_VIEWPORT = 
{ width: 1366, height: 900 }.freeze
DEFAULT_USER_AGENT = 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " \
"(KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
DEFAULT_ACCEPT_LANGUAGE = 
"en-US,en;q=0.9"
SOCIAL_LOGIN_PHASE_WAIT = 
0.3
POST_CONSENT_IDLE_TIMEOUT = 
3.0
CONTENT_READY_MIN_LENGTH = 
200
SPA_HYDRATION_TIMEOUT = 
2.0
SPA_HYDRATION_POLL = 
0.15
PRE_EXTRACTION_SETTLE_WAIT = 
0.25
HEAVY_SCRIPT_COUNT_THRESHOLD = 
20
2
2.0

Constants included from Stabilization::PageFlow

Stabilization::PageFlow::PAGE_FLOW_STABILIZATION_PROFILES, Stabilization::PageFlow::POST_GENERIC_STABILIZATION_PROFILES

Constants included from SiteStabilization::SocialPlatforms

SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES

Constants included from SiteStabilization::CommunityAndMarketplace

SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES

Instance Method Summary collapse

Constructor Details

#initialize(timeout: 20, wait: 0.75, wait_for_idle: true, idle_duration: 0.35, viewport: DEFAULT_VIEWPORT, user_agent: DEFAULT_USER_AGENT, accept_language: DEFAULT_ACCEPT_LANGUAGE, browser_path: nil, browser_options: nil) ⇒ Browser

Returns a new instance of Browser.



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/fetch_util/browser.rb', line 44

def initialize(timeout: 20, wait: 0.75, wait_for_idle: true, idle_duration: 0.35,
               viewport: DEFAULT_VIEWPORT, user_agent: DEFAULT_USER_AGENT,
               accept_language: DEFAULT_ACCEPT_LANGUAGE, browser_path: nil,
               browser_options: nil)
  @timeout = timeout.to_f
  @wait = wait.to_f
  @wait_for_idle = wait_for_idle
  @idle_duration = idle_duration.to_f
  @viewport = DEFAULT_VIEWPORT.merge(symbolize_hash(viewport || {}))
  @user_agent = user_agent
  @accept_language = accept_language
  @default_headers = {
    "User-Agent" => @user_agent,
    "Accept-Language" => @accept_language
  }.freeze
  @browser_path = browser_path || ENV["BROWSER_PATH"] || BROWSER_CANDIDATES.find { |path| File.executable?(path) }
  @full_browser = @browser_path && !@browser_path.include?("headless_shell")
  default_opts = { "no-sandbox": nil }
  # Use newer headless mode with the full browser binary for closer runtime
  # parity with standard Chromium. Also disable Ferrum's
  # `enable-automation` flag to reduce tool-specific browser-state
  # differences during extraction.
  if @full_browser
    default_opts["headless"] = "new"
    default_opts["enable-automation"] = false # override Ferrum default
  end
  @browser_options = default_opts.merge(browser_options || {})
  @navigator_patch = build_navigator_patch
  @ferrum = nil
  @mutex = Mutex.new
end

Instance Method Details

#quit ⇒ Object

Shut down the underlying Chromium process. Safe to call multiple times or when no browser has been started yet. After quit, a subsequent with_page call will transparently launch a new process.



96
97
98
99
100
101
 
# File 'lib/fetch_util/browser.rb', line 96

def quit
  @mutex.synchronize do
    @ferrum&.quit
    @ferrum = nil
  end
end

#with_page(url) ⇒ Object

Navigate to url in a fresh browser tab, stabilize the page, then yield the Ferrum::Page to the caller. The page is closed after the block returns (or on error), but the underlying Chromium process is kept alive for reuse by subsequent calls.



80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/fetch_util/browser.rb', line 80

def with_page(url)
  raise BrowserError, "No Chromium browser found. Set BROWSER_PATH or install Chromium." unless @browser_path

  normalized_url = FetchUtil.normalize_url(url)
  page = load_page_with_retry(ensure_browser, normalized_url)
  sleep PRE_EXTRACTION_SETTLE_WAIT if heavy_script_page?(page, normalized_url)
  yield page
rescue Ferrum::Error => e
  raise BrowserError, e.message
ensure
  page&.close
end