Class: AgentSandbox::BrowserTools::ReadImage

Inherits:
Base
  • Object
show all
Defined in:
lib/agent_sandbox/browser_tools.rb

Overview

Downloads an image URL from inside the sandbox and asks a vision model to read it. Ideal for brochures/flipbooks where product pages are served as discoverable <img> URLs — higher resolution than a viewport screenshot, and skips browser chrome entirely.

Instance Method Summary collapse

Methods inherited from Base

#run_ab

Constructor Details

#initialize(sandbox, vision_model:) ⇒ ReadImage

Returns a new instance of ReadImage.



277
278
279
280
 
# File 'lib/agent_sandbox/browser_tools.rb', line 277

def initialize(sandbox, vision_model:)
  @vision_model = vision_model
  super(sandbox)
end

Instance Method Details

#execute(url:, focus: nil) ⇒ Object 



282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
 
# File 'lib/agent_sandbox/browser_tools.rb', line 282

def execute(url:, focus: nil)
  # Fetch through the current page's fetch() so cookies + origin
  # headers from the live session are sent (image URLs on logged-in
  # pages commonly require them). Fall back to a direct download if
  # there's no page context or the in-page fetch refuses (CORS etc).
  bytes, content_type, session_error = fetch_via_session(url)
  if bytes.nil?
    bytes, content_type, curl_error = fetch_via_curl(url)
    if bytes.nil?
      return { error: "download failed", url: url,
               session_error: session_error, curl_error: curl_error }
    end
  end

  unless content_type.to_s.start_with?("image/")
    return { error: "not an image", url: url, content_type: content_type,
             bytes: bytes.bytesize,
             hint: "response wasn't image/* — often a redirect to an HTML login/CAPTCHA page. Try opening the URL in the browser first to authenticate, then retry." }
  end

  description = VisionSupport.read_image_bytes(
    bytes, extension: content_type_to_ext(content_type) || "img",
    focus: focus, vision_model: @vision_model
  )
  { description: description, url: url, content_type: content_type,
    vision_model: @vision_model }
end