Skip to content
Kward Search API index

Class: Kward::WebFetch

Inherits:
Object
  • Object
show all
Defined in:
lib/kward/tools/search/web_fetch.rb

Overview

Fetches specific web resources for agent research workflows.

Constant Summary collapse

DEFAULT_MAX_BYTES = 
16 * 1024
MAX_MAX_BYTES = 
128 * 1024
MAX_REDIRECTS = 
5
MAX_DOWNLOAD_BYTES = 
2 * 1024 * 1024
HTTP_TIMEOUT_SECONDS = 
10
50

Instance Method Summary collapse

Constructor Details

#initialize(http_client: WebSearch::NetHttpClient.new) ⇒ WebFetch

Creates a fetcher for web content and raw resources.



18
19
20
 
# File 'lib/kward/tools/search/web_fetch.rb', line 18

def initialize(http_client: WebSearch::NetHttpClient.new)
  @http_client = http_client
end

Instance Method Details

#fetch_content(args = nil, cancellation: nil, **keyword_args) ⇒ Object

Fetches a URL and extracts readable text for human-facing pages.



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/kward/tools/search/web_fetch.rb', line 23

def fetch_content(args = nil, cancellation: nil, **keyword_args)
  args ||= keyword_args
  cancellation&.raise_if_cancelled!
  url = args_value(args, "url").to_s.strip
  return "Error: url is required" if url.empty?

  max_bytes = bounded_max_bytes(args_value(args, "max_bytes") || args_value(args, "maxBytes"))
  extract = normalize_extract(args_value(args, "extract") || "auto")
  return "Error: extract must be one of: auto, text, markdown" unless extract

  response = fetch_url(url, max_bytes: MAX_DOWNLOAD_BYTES, cancellation: cancellation)
  return response if response.is_a?(String)
  return "Error: response exceeds #{MAX_DOWNLOAD_BYTES} byte download limit" if response[:truncated]

  body = response[:body].to_s
  content_type = header_value(response[:headers], "content-type")
  text = extract_readable_text(body, content_type: content_type, mode: extract, base_url: response[:url])
  text, truncated = truncate_bytes(text, max_bytes)

  [
    "# Fetched content",
    "- URL: #{response[:url]}",
    "- Content type: #{content_type.empty? ? "unknown" : content_type}",
    "- Downloaded bytes: #{body.bytesize}",
    "- Bytes returned: #{text.bytesize}",
    "- Truncated: #{truncated ? "yes" : "no"}",
    "",
    text.empty? ? "(No readable text extracted.)" : text
  ].join("\n")
rescue Cancellation::CancelledError
  raise
rescue StandardError => e
  "Error: fetch_content failed: #{e.message}"
end

#fetch_raw(args = nil, cancellation: nil, **keyword_args) ⇒ Object

Fetches a URL and returns bounded raw response content.



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/kward/tools/search/web_fetch.rb', line 59

def fetch_raw(args = nil, cancellation: nil, **keyword_args)
  args ||= keyword_args
  cancellation&.raise_if_cancelled!
  url = args_value(args, "url").to_s.strip
  return "Error: url is required" if url.empty?

  max_bytes = bounded_max_bytes(args_value(args, "max_bytes") || args_value(args, "maxBytes"))
  accept = args_value(args, "accept").to_s.strip
  response = fetch_url(url, max_bytes: max_bytes, accept: accept.empty? ? "*/*" : accept, cancellation: cancellation)
  return response if response.is_a?(String)

  body = response[:body].to_s
  content_type = header_value(response[:headers], "content-type")
  body = "#{body.to_s.scrub}\n... truncated to #{max_bytes} bytes" if response[:truncated]
  [
    "# Fetched raw content",
    "- URL: #{response[:url]}",
    "- Content type: #{content_type.empty? ? "unknown" : content_type}",
    "- Bytes returned: #{response[:body].to_s.bytesize}",
    "- Truncated: #{response[:truncated] ? "yes" : "no"}",
    "",
    body
  ].join("\n")
rescue Cancellation::CancelledError
  raise
rescue StandardError => e
  "Error: fetch_raw failed: #{e.message}"
end