Class: Html2rss::RequestService::BotasaurusContract::ParsedResponse

Inherits:
Object
  • Object
show all
Defined in:
lib/html2rss/request_service/botasaurus_contract.rb

Overview

Parsed Botasaurus response wrapper.

Constant Summary collapse

DEFAULT_HEADERS =

Fallback headers when upstream omits response headers.

{ 'content-type' => 'text/html' }.freeze
MAX_XHR_BODY_BYTES =

Per-body size cap for captured XHR JSON (defense-in-depth vs upstream).

500_000
MAX_XHR_AGGREGATE_BYTES =

Aggregate size cap across all captured XHR bodies.

2_000_000

Instance Method Summary collapse

Constructor Details

#initialize(payload:, transport_status:) ⇒ ParsedResponse

Returns a new instance of ParsedResponse.

Parameters:

  • payload (Hash{String => Object}) —

    parsed Botasaurus response payload

  • transport_status (Integer) —

    HTTP status returned by Botasaurus



63
64
65
66
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 63

def initialize(payload:, transport_status:)
  @payload = payload
  @transport_status = transport_status
end

Instance Method Details

#challenge_block? ⇒ Boolean

Returns true when upstream classified request as challenge blocked.

Returns:

  • (Boolean) —

    true when upstream classified request as challenge blocked



69
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 69

def challenge_block? = error_category == 'challenge_block'

#challenge_message ⇒ String

Returns normalized challenge error message.

Returns:

  • (String) —

    normalized challenge error message



77
78
79
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 77

def challenge_message
  error || 'Botasaurus challenge block detected.'
end

#final_url ⇒ String?

Returns final URL reported by upstream.

Returns:

  • (String, nil) —

    final URL reported by upstream



114
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 114

def final_url = payload['final_url']

#headers ⇒ Hash{String => String}

Returns normalized response headers.

Returns:

  • (Hash{String => String}) —

    normalized response headers



100
101
102
103
104
105
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 100

def headers
  raw_headers = payload['headers']
  return DEFAULT_HEADERS.dup unless raw_headers.is_a?(Hash) && raw_headers.any?

  raw_headers.to_h { |key, value| [key.to_s, value.to_s] }
end

#html ⇒ String

Returns rendered HTML body from Botasaurus.

Returns:

  • (String) —

    rendered HTML body from Botasaurus

Raises:



92
93
94
95
96
97
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 92

def html
  value = payload['html']
  raise BotasaurusServiceError, "Botasaurus response missing required 'html' field" if value.nil?

  value.to_s
end

#status ⇒ Integer

Returns resolved status code (payload status_code or transport status).

Returns:

  • (Integer) —

    resolved status code (payload status_code or transport status)



108
109
110
111
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 108

def status
  status_code = payload['status_code']
  status_code.is_a?(Integer) ? status_code : transport_status
end

#transport_meta ⇒ Hash{String => Object}

Returns allowlisted upstream telemetry (frozen).

Returns:

  • (Hash{String => Object}) —

    allowlisted upstream telemetry (frozen)



117
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 117

def transport_meta = payload.slice(*META_KEYS).compact.freeze

#upstream_failure? ⇒ Boolean

Returns true when upstream returned non-200 or an error payload.

Returns:

  • (Boolean) —

    true when upstream returned non-200 or an error payload



72
73
74
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 72

def upstream_failure?
  transport_status != 200 || status != 200 || error_message?
end

#upstream_failure_message ⇒ String

Returns actionable upstream failure summary.

Returns:

  • (String) —

    actionable upstream failure summary



82
83
84
85
86
87
88
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 82

def upstream_failure_message
  details = ["status=#{status}"]
  details << "error_category=#{error_category}" if error_category
  details << "error=#{error}" if error
  details << "request_id=#{request_id}" if request_id
  "Botasaurus scrape failed (#{details.join(', ')})."
end

#xhr_responses ⇒ Array<Hash{String => Object}>

Returns size-capped XHR/fetch captures (absent → []).

Returns:

  • (Array<Hash{String => Object}>) —

    size-capped XHR/fetch captures (absent → [])



120
121
122
123
124
125
126
127
128
129
130
131
132
 
# File 'lib/html2rss/request_service/botasaurus_contract.rb', line 120

def xhr_responses
  raw = payload['xhr_responses']
  return [] unless raw.is_a?(Array)

  aggregate_bytes = 0
  raw.filter_map do |entry|
    normalized = normalize_xhr_entry(entry)
    next unless normalized
    next unless (aggregate_bytes = advance_xhr_budget(normalized, aggregate_bytes))

    normalized
  end
end