Class: FetchUtil::SearchTransport

Inherits:
Object
  • Object
show all
Defined in:
lib/fetch_util/search_transport.rb

Overview

Fetches and parses supported search result pages without involving the browser fetcher.

Defined Under Namespace

Classes: Candidate, HttpClient, HttpFailure, HttpResponse, SourceResponse

Constant Summary collapse

SOURCES = 
{
  "brave" => { url: "https://search.brave.com/search?q=%{query}", hosts: %w[search.brave.com] },
  "bing" => { url: "https://www.bing.com/search?q=%{query}&setlang=en-US&cc=US", hosts: %w[www.bing.com cn.bing.com] },
  "duckduckgo" => { url: "https://html.duckduckgo.com/html/?q=%{query}", hosts: %w[html.duckduckgo.com] },
  "google" => { url: "https://www.google.com/search?q=%{query}", hosts: %w[www.google.com] },
  "ecosia" => { url: "https://www.ecosia.org/search?q=%{query}", hosts: %w[www.ecosia.org] }
}.freeze
DEFAULT_TIMEOUT = 
10.0
FAILURE_REASONS = 
%w[challenge failed host http_status parse query_mismatch redirect size timeout].freeze
RELEVANCE_HEALTH_CANDIDATE_COUNT = 
3
QUERY_FUNCTION_WORDS = 
%w[
  a an and are at be been being but by can could did do does for from had has have how i if in is it me my no not of on or our shall
  should that the their them then there these they this to was we were what when where which who why will with would you your
].freeze
WRAPPER_HOSTS = 
{
  "bing" => %w[bing.com www.bing.com cn.bing.com],
  "duckduckgo" => %w[duckduckgo.com www.duckduckgo.com html.duckduckgo.com],
  "google" => %w[google.com www.google.com]
}.freeze

Instance Method Summary collapse

Constructor Details

#initialize(sources: SOURCES.keys, timeout: DEFAULT_TIMEOUT, clock: nil, http_client: nil, html_parser: nil) ⇒ SearchTransport

Returns a new instance of SearchTransport.

Raises:

  • (ArgumentError) 


52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/fetch_util/search_transport.rb', line 52

def initialize(sources: SOURCES.keys, timeout: DEFAULT_TIMEOUT, clock: nil, http_client: nil, html_parser: nil)
  @sources = sources.map(&:to_s).freeze
  unknown = @sources - SOURCES.keys
  raise ArgumentError, "unknown search sources: #{unknown.join(", ")}" if unknown.any?

  @timeout = Float(timeout)
  raise ArgumentError, "timeout must be positive" unless @timeout.positive?

  @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
  @http_client = http_client || HttpClient.new(clock: @clock)
  @html_parser = html_parser || ->(body) { Nokogiri::HTML(body) }
end

Instance Method Details

#search(query) ⇒ Object

Raises:

  • (ArgumentError) 


65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/fetch_util/search_transport.rb', line 65

def search(query)
  query = query.to_s.strip
  raise ArgumentError, "query must not be empty" if query.empty?

  deadline = clock.call + timeout
  responses = Array.new(sources.length)
  threads = sources.each_with_index.map do |source, index|
    Thread.new { responses[index] = search_source(source, query, deadline) }
  end
  threads.each(&:join)
  responses
end