Class: Ask::WebFetch::Backends::Jina

Inherits:
Ask::WebFetch::Backend show all
Defined in:
lib/ask/web_fetch/backends/jina.rb

Overview

Jina Reader free tier: GET https://r.jina.ai/.

Free without a key (~20 req/min per IP). Set JINA_API_KEY for higher rate limits. The endpoint runs headless Chromium, so it renders JS pages that the Local backend cannot.

Constant Summary collapse

BASE_URL = 
'https://r.jina.ai'
OPEN_TIMEOUT = 
5
READ_TIMEOUT = 
30

Constants inherited from Ask::WebFetch::Backend

Ask::WebFetch::Backend::CHALLENGE_RE, Ask::WebFetch::Backend::MIN_CONTENT_LENGTH, Ask::WebFetch::Backend::USER_AGENT

Instance Method Summary collapse

Methods inherited from Ask::WebFetch::Backend

backend_name

Instance Method Details

#fetch(url) ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/ask/web_fetch/backends/jina.rb', line 20

def fetch(url)
  uri = URI("#{BASE_URL}/#{url}")
  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = true
  http.open_timeout = OPEN_TIMEOUT
  http.read_timeout = READ_TIMEOUT
  req = Net::HTTP::Get.new(uri)
  req['User-Agent'] = USER_AGENT
  req['Accept'] = 'text/markdown, text/plain, text/html'
  req['Authorization'] = "Bearer #{ENV['JINA_API_KEY']}" if ENV['JINA_API_KEY']

  res = http.request(req)
  case res.code
  when '200'
    body = res.body.to_s
    raise FetchError, 'challenge page from Jina' if challenge_page?(body)
    raise EmptyContentError, 'empty response from Jina' unless usable_content?(body)

    { title: nil, content: body.strip }
  when '429'
    raise FetchError, 'rate limited by Jina (429)'
  when '401', '403'
    raise FetchError, "Jina access error (#{res.code})"
  else
    raise FetchError, "Jina returned #{res.code}"
  end
rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED,
       Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
  raise FetchError, "Jina #{e.class}: #{e.message}"
end