Module: Html2rss::MCP::Inspect

Defined in:
lib/html2rss/mcp/inspect.rb

Overview

Diagnostic inspect path (not Capture ownership). Fetches once for SST/scraper stats and recon facts (final URL, status, scheme downgrade, native feed hints).

Class Method Summary collapse

Class Method Details

.alternate_feeds_from(parsed) ⇒ Array<Hash{Symbol => String}>

Parameters:

  • parsed (Object)

Returns:

  • (Array<Hash{Symbol => String}>) 


233
234
235
236
237
 
# File 'lib/html2rss/mcp/inspect.rb', line 233

def alternate_feeds_from(parsed)
  return [] unless parsed.is_a?(Nokogiri::HTML::Document)

  Html::FeedLink.from_document(parsed).map { |link| { href: link.href, mime_type: link.mime_type } }
end

.call(url:, strategy: :auto) ⇒ Hash

Parameters:

  • url (String)
  • strategy (String, Symbol) (defaults to: :auto)

Returns:

  • (Hash) 


15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/html2rss/mcp/inspect.rb', line 15

def call(url:, strategy: :auto) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
  resolved = FeedPipeline::StrategyPlan.concrete_for_diagnostic(strategy)
  response = fetch_response(url, resolved)
  parsed = response.parsed_body

  result = recon_fields(url, response, parsed).merge(
    strategy: resolved,
    content_type: response.content_type,
    html_response: response.html_response?,
    scraper_eligibility: scraper_info(parsed),
    sst_stats: sst_stats_from(response)
  )

  if response.html_response?
    sst = sst_document(response)
    if sst
      result[:sst] = {
        node_count: sst.node_count,
        degraded: sst.degraded,
        segment_stats: segment_stats(sst, url)
      }
    end
  end

  blocked = Html2rss::RequestService::BlockedSurface.interstitial_signature_for(response.body)
  result[:blocked_surface] = blocked[:key].to_s if blocked
  result[:xhr_capture] = xhr_capture_info(response) if resolved == :botasaurus
  merge_admission_diagnostics!(result, response)

  result
end

.discover_segments(sst, url) ⇒ Array

Parameters:

Returns:

  • (Array) 


189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/html2rss/mcp/inspect.rb', line 189

def discover_segments(sst, url)
  link_resolver = Scoring::LinkResolver.new(url)
  AutoSource::Segmenter.call(
    sst,
    base_url: url,
    strategy: :list,
    link_resolver:
  )
rescue StandardError
  []
end

.fetch_response(url, strategy) ⇒ Html2rss::RequestService::Response

Parameters:

  • url (String)
  • strategy (Symbol)

Returns:



112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/html2rss/mcp/inspect.rb', line 112

def fetch_response(url, strategy) # rubocop:disable Metrics/MethodLength -- session construction
  raw_config = Config.auto_source_config(
    url:,
    request_controls: Config::RequestControls.from_shortcut(strategy:)
  )
  raw_config[:strategy] = strategy
  config = Config.from_hash(raw_config)
  resources = FeedPipeline::RuntimePolicy.resources_for(config)
  session = RequestSession.build(
    config:,
    strategy: config.strategy,
    budget: resources.budget,
    policy: resources.policy
  )
  session.fetch_initial_response
end

.merge_admission_diagnostics!(result, response) ⇒ void

This method returns an undefined value.

Surfaces Cleanup admission_drops without re-running full AutoSource discovery. Uses articles already extractable from a cheap AutoSource pass only when HTML.

Parameters:



54
55
56
57
58
59
60
61
62
 
# File 'lib/html2rss/mcp/inspect.rb', line 54

def merge_admission_diagnostics!(result, response)
  return unless response.html_response?

  source = AutoSource.new(response, AutoSource::DEFAULT_CONFIG.merge(limit: 10))
  articles = source.articles
  result[:articles_count] = articles.size
  drops = source.admission_drops
  result[:admission_drops] = drops if drops.any?
end

.recon_fields(requested_url, response, parsed) ⇒ Hash

Parameters:

Returns:

  • (Hash) 


207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/html2rss/mcp/inspect.rb', line 207

def recon_fields(requested_url, response, parsed)
  requested = Url.from_absolute(requested_url)
  final = response.url

  {
    requested_url: requested.to_s,
    final_url: final.to_s,
    status: response.status,
    scheme_downgrade: scheme_downgrade?(requested, final),
    alternate_feeds: alternate_feeds_from(parsed)
  }
end

.redacted_endpoint(entry) ⇒ String?

Returns scheme+host+path only.

Parameters:

  • entry (Hash) —

    captured response hash

Returns:

  • (String, nil) —

    scheme+host+path only



81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/html2rss/mcp/inspect.rb', line 81

def redacted_endpoint(entry)
  raw = entry['url'] || entry[:url]
  return unless raw

  uri = URI.parse(raw.to_s)
  return unless uri.scheme && uri.host

  "#{uri.scheme}://#{uri.host}#{uri.path}"
rescue URI::InvalidURIError
  nil
end

.scheme_downgrade?(requested, final) ⇒ Boolean

Returns true when the fetch downgraded https to http.

Parameters:

Returns:

  • (Boolean) —

    true when the fetch downgraded https to http



225
226
227
 
# File 'lib/html2rss/mcp/inspect.rb', line 225

def scheme_downgrade?(requested, final)
  requested.scheme == 'https' && final.scheme == 'http'
end

.scraper_info(parsed) ⇒ Array<String>, Hash

Parameters:

  • parsed (Object) —

    parsed response body

Returns:

  • (Array<String>, Hash) 


133
134
135
136
137
138
139
140
141
 
# File 'lib/html2rss/mcp/inspect.rb', line 133

def scraper_info(parsed)
  return { error: 'Response is not HTML' } unless parsed.is_a?(Nokogiri::HTML::Document)

  begin
    Html2rss::AutoSource::Scraper.from(parsed).map(&:name)
  rescue Html2rss::AutoSource::Scraper::NoScraperFound => error
    { none_found: error.category.to_s }
  end
end

.segment_stats(sst, url) ⇒ Hash

Parameters:

Returns:

  • (Hash) 


173
174
175
176
177
178
179
180
181
182
 
# File 'lib/html2rss/mcp/inspect.rb', line 173

def segment_stats(sst, url)
  segments = discover_segments(sst, url)
  return { found: 0 } if segments.empty?

  {
    found: segments.size,
    strategies: segments.map(&:strategy).uniq,
    sample_paths: segments.first(5).map { |s| s.root_node.tag_path }
  }
end

.sst_document(response) ⇒ Html2rss::SST::Document?

Parameters:

Returns:



162
163
164
165
166
 
# File 'lib/html2rss/mcp/inspect.rb', line 162

def sst_document(response)
  Html2rss::SST::Normalizer.call(response.body)
rescue ArgumentError
  nil
end

.sst_stats_from(response) ⇒ Hash?

Parameters:

Returns:

  • (Hash, nil) 


147
148
149
150
151
152
153
154
155
156
 
# File 'lib/html2rss/mcp/inspect.rb', line 147

def sst_stats_from(response)
  return nil unless response.html_response?

  doc = sst_document(response)
  return nil unless doc

  { node_count: doc.node_count, degraded: doc.degraded }
rescue StandardError
  nil
end

.xhr_candidate_articles?(entry) ⇒ Boolean

Parameters:

  • entry (Hash) —

    captured response hash

Returns:

  • (Boolean) 


97
98
99
100
101
102
103
104
105
 
# File 'lib/html2rss/mcp/inspect.rb', line 97

def xhr_candidate_articles?(entry)
  body = entry['body'] || entry[:body]
  return false unless body.is_a?(String)

  document = JSON.parse(body, symbolize_names: true)
  AutoSource::Scraper::JsonState::CandidateDetector.candidate_array?(document)
rescue JSON::ParserError
  false
end

.xhr_capture_info(response) ⇒ Hash

Returns redacted XHR capture diagnostics (no query strings).

Parameters:

Returns:

  • (Hash) —

    redacted XHR capture diagnostics (no query strings)



68
69
70
71
72
73
74
75
 
# File 'lib/html2rss/mcp/inspect.rb', line 68

def xhr_capture_info(response)
  captured = response.captured_responses
  {
    count: captured.size,
    sample_endpoints: captured.first(5).filter_map { |entry| redacted_endpoint(entry) },
    candidate_articles: captured.any? { |entry| xhr_candidate_articles?(entry) }
  }
end