Class: FetchUtil::SearchTransport

Inherits:
Object
  • Object
show all
Defined in:
lib/fetch_util/search_transport.rb

Overview

Fetches and parses supported search result pages without involving the browser fetcher.

Defined Under Namespace

Classes: Candidate, HttpClient, HttpFailure, HttpResponse, SourceResponse

Constant Summary collapse

SOURCES = 
{
  "brave" => { url: "https://search.brave.com/search?q=%{query}", hosts: %w[search.brave.com] },
  "bing" => { url: "https://www.bing.com/search?q=%{query}&setlang=en-US&cc=US", hosts: %w[www.bing.com cn.bing.com] },
  "duckduckgo" => { url: "https://html.duckduckgo.com/html/?q=%{query}", hosts: %w[html.duckduckgo.com] },
  "google" => { url: "https://www.google.com/search?q=%{query}", hosts: %w[www.google.com] },
  "ecosia" => { url: "https://www.ecosia.org/search?q=%{query}", hosts: %w[www.ecosia.org] },
  "yahoo" => { url: "https://search.yahoo.com/search?p=%{query}", hosts: %w[search.yahoo.com] }
}.freeze
DEFAULT_TIMEOUT = 
10.0
YAHOO_RECOVERY_ATTEMPTS = 
2
FAILURE_REASONS = 
%w[challenge failed host http_status parse query_mismatch redirect size timeout].freeze
RELEVANCE_HEALTH_CANDIDATE_COUNT = 
3
QUERY_FUNCTION_WORDS = 
%w[
  a an and are at be been being but by can could did do does for from had has have how i if in is it me my no not of on or our shall
  should that the their them then there these they this to was we were what when where which who why will with would you your
].freeze
SCOPED_QUERY_TERM = 
/(?:\A|\s)[+-]?[[:alpha:]][[:alnum:]_-]*:(?:"[^"]*"|'[^']*'|\S+)/
NEGATED_QUERY_TERM = 
/(?:\A|\s)-\S+/
WRAPPER_HOSTS = 
{
  "bing" => %w[bing.com www.bing.com cn.bing.com],
  "duckduckgo" => %w[duckduckgo.com www.duckduckgo.com html.duckduckgo.com],
  "google" => %w[google.com www.google.com],
  "yahoo" => %w[r.search.yahoo.com]
}.freeze

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(sources: SOURCES.keys, timeout: DEFAULT_TIMEOUT, clock: nil, http_client: nil, html_parser: nil) ⇒ SearchTransport

Returns a new instance of SearchTransport.

Raises:

  • (ArgumentError) 


57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/fetch_util/search_transport.rb', line 57

def initialize(sources: SOURCES.keys, timeout: DEFAULT_TIMEOUT, clock: nil, http_client: nil, html_parser: nil)
  @sources = sources.map(&:to_s).freeze
  unknown = @sources - SOURCES.keys
  raise ArgumentError, "unknown search sources: #{unknown.join(", ")}" if unknown.any?

  @timeout = Float(timeout)
  raise ArgumentError, "timeout must be positive" unless @timeout.positive?

  @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
  @http_client = http_client || HttpClient.new(clock: @clock)
  @html_parser = html_parser || ->(body) { Nokogiri::HTML(body) }
end

Class Method Details

.candidate_matches_query?(query, candidate) ⇒ Boolean

Returns:

  • (Boolean) 


76
77
78
79
80
81
82
 
# File 'lib/fetch_util/search_transport.rb', line 76

def self.candidate_matches_query?(query, candidate)
  query_terms = meaningful_query_terms(normalized_query_text(query))
  return true if query_terms.length < 3

  candidate_terms = normalized_terms("#{candidate.title} #{candidate.snippet}")
  (query_terms & candidate_terms).length >= 2
end

.candidates_match_query?(query, candidates) ⇒ Boolean

Returns:

  • (Boolean) 


70
71
72
73
74
 
# File 'lib/fetch_util/search_transport.rb', line 70

def self.candidates_match_query?(query, candidates)
  candidates.first(RELEVANCE_HEALTH_CANDIDATE_COUNT).any? do |candidate|
    candidate_matches_query?(query, candidate)
  end
end

Instance Method Details

#search(query, timeout: @timeout) ⇒ Object

Raises:

  • (ArgumentError) 


84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/fetch_util/search_transport.rb', line 84

def search(query, timeout: @timeout)
  query = query.to_s.strip
  raise ArgumentError, "query must not be empty" if query.empty?

  request_timeout = Float(timeout)
  raise ArgumentError, "timeout must be positive" unless request_timeout.positive?

  deadline = clock.call + request_timeout
  responses = Array.new(sources.length)
  threads = sources.each_with_index.map do |source, index|
    Thread.new { responses[index] = search_source(source, query, deadline) }
  end
  threads.each(&:join)
  responses
end