Module: Html2rss

Defined in:: lib/html2rss.rb,
lib/html2rss/cli.rb,
lib/html2rss/sst.rb,
lib/html2rss/url.rb,
lib/html2rss/html.rb,
lib/html2rss/error.rb,
lib/html2rss/config.rb,
lib/html2rss/status.rb,
lib/html2rss/article.rb,
lib/html2rss/channel.rb,
lib/html2rss/scoring.rb,
lib/html2rss/version.rb,
lib/html2rss/defaults.rb,
lib/html2rss/sst/node.rb,
lib/html2rss/sst/tags.rb,
lib/html2rss/sst/text.rb,
lib/html2rss/hash_util.rb,
lib/html2rss/selectors.rb,
lib/html2rss/sst/attrs.rb,
lib/html2rss/sst/index.rb,
lib/html2rss/auto_source.rb,
lib/html2rss/feed_result.rb,
lib/html2rss/feed_builder.rb,
lib/html2rss/sst/document.rb,
lib/html2rss/config/schema.rb,
lib/html2rss/feed_pipeline.rb,
lib/html2rss/scoring/score.rb,
lib/html2rss/html/navigator.rb,
lib/html2rss/html/rendering.rb,
lib/html2rss/scoring/engine.rb,
lib/html2rss/sst/normalizer.rb,
lib/html2rss/request_service.rb,
lib/html2rss/request_session.rb,
lib/html2rss/config/validator.rb,
lib/html2rss/feed_builder/rss.rb,
lib/html2rss/link_destination.rb,
lib/html2rss/article/enclosure.rb,
lib/html2rss/html/article_rules.rb,
lib/html2rss/auto_source/cleanup.rb,
lib/html2rss/auto_source/scraper.rb,
lib/html2rss/auto_source/segment.rb,
lib/html2rss/scoring/observation.rb,
lib/html2rss/article/deduplicator.rb,
lib/html2rss/scoring/anchor_score.rb,
lib/html2rss/selectors/extractors.rb,
lib/html2rss/selectors/item_scope.rb,
lib/html2rss/auto_source/segmenter.rb,
lib/html2rss/config/dynamic_params.rb,
lib/html2rss/request_session/pager.rb,
lib/html2rss/scoring/link_resolver.rb,
lib/html2rss/config/request_headers.rb,
lib/html2rss/feed_builder/json_feed.rb,
lib/html2rss/html/article_extractor.rb,
lib/html2rss/request_service/budget.rb,
lib/html2rss/request_service/policy.rb,
lib/html2rss/scoring/cluster_scorer.rb,
lib/html2rss/scoring/ranked_segment.rb,
lib/html2rss/config/request_controls.rb,
lib/html2rss/html/article_rules/date.rb,
lib/html2rss/request_service/context.rb,
lib/html2rss/auto_source/scraper/html.rb,
lib/html2rss/html/article_rules/image.rb,
lib/html2rss/request_service/response.rb,
lib/html2rss/request_service/strategy.rb,
lib/html2rss/selectors/extractors/href.rb,
lib/html2rss/selectors/extractors/html.rb,
lib/html2rss/selectors/extractors/text.rb,
lib/html2rss/selectors/post_processors.rb,
lib/html2rss/auto_source/scraper/schema.rb,
lib/html2rss/auto_source/segmenter/list.rb,
lib/html2rss/config/selectors_validator.rb,
lib/html2rss/html/sst_article_extractor.rb,
lib/html2rss/request_session/pager/base.rb,
lib/html2rss/scoring/container_assessor.rb,
lib/html2rss/auto_source/scraper/sitemap.rb,
lib/html2rss/config/auto_source_contract.rb,
lib/html2rss/feed_builder/json_feed/item.rb,
lib/html2rss/feed_builder/rss/stylesheet.rb,
lib/html2rss/feed_pipeline/auto_fallback.rb,
lib/html2rss/feed_pipeline/strategy_plan.rb,
lib/html2rss/html/article_rules/category.rb,
lib/html2rss/html/rendering/pdf_renderer.rb,
lib/html2rss/selectors/extractors/static.rb,
lib/html2rss/config/multiple_feeds_config.rb,
lib/html2rss/feed_pipeline/runtime_policy.rb,
lib/html2rss/html/article_rules/enclosure.rb,
lib/html2rss/request_session/pager/offset.rb,
lib/html2rss/auto_source/scraper/microdata.rb,
lib/html2rss/auto_source/segmenter/cluster.rb,
lib/html2rss/html/navigator/text_extractor.rb,
lib/html2rss/html/rendering/audio_renderer.rb,
lib/html2rss/html/rendering/image_renderer.rb,
lib/html2rss/html/rendering/media_renderer.rb,
lib/html2rss/html/rendering/video_renderer.rb,
lib/html2rss/link_destination/noise_policy.rb,
lib/html2rss/request_service/network_guard.rb,
lib/html2rss/auto_source/scraper/json_state.rb,
lib/html2rss/auto_source/segmenter/semantic.rb,
lib/html2rss/feed_builder/item_presentation.rb,
lib/html2rss/request_service/response_guard.rb,
lib/html2rss/request_session/pager/rel_next.rb,
lib/html2rss/selectors/extractors/attribute.rb,
lib/html2rss/selectors/post_processors/base.rb,
lib/html2rss/selectors/post_processors/gsub.rb,
lib/html2rss/auto_source/scraper/meta_oembed.rb,
lib/html2rss/request_service/blocked_surface.rb,
lib/html2rss/auto_source/scraper/schema/thing.rb,
lib/html2rss/link_destination/path_classifier.rb,
lib/html2rss/link_destination/text_classifier.rb,
lib/html2rss/request_service/faraday_strategy.rb,
lib/html2rss/request_service/puppet_commander.rb,
lib/html2rss/auto_source/scraper/microformats2.rb,
lib/html2rss/auto_source/scraper/semantic_html.rb,
lib/html2rss/auto_source/scraper/wordpress_api.rb,
lib/html2rss/html/rendering/escaped_attributes.rb,
lib/html2rss/request_session/pager/json_cursor.rb,
lib/html2rss/selectors/object_to_xml_converter.rb,
lib/html2rss/auto_source/scraper/sitemap/parser.rb,
lib/html2rss/auto_source/segmenter/primary_link.rb,
lib/html2rss/html/rendering/description_builder.rb,
lib/html2rss/link_destination/destination_facts.rb,
lib/html2rss/request_session/pager/url_template.rb,
lib/html2rss/selectors/post_processors/template.rb,
lib/html2rss/html/article_extractor/id_generator.rb,
lib/html2rss/request_service/botasaurus_contract.rb,
lib/html2rss/request_service/botasaurus_strategy.rb,
lib/html2rss/request_service/local_file_strategy.rb,
lib/html2rss/selectors/post_processors/parse_uri.rb,
lib/html2rss/selectors/post_processors/substring.rb,
lib/html2rss/auto_source/scraper/schema/item_list.rb,
lib/html2rss/request_service/browserless_strategy.rb,
lib/html2rss/selectors/post_processors/parse_time.rb,
lib/html2rss/html/article_extractor/date_extractor.rb,
lib/html2rss/request_session/pager/custom_selector.rb,
lib/html2rss/html/article_extractor/image_extractor.rb,
lib/html2rss/selectors/post_processors/sanitize_html.rb,
lib/html2rss/html/article_extractor/heading_extractor.rb,
lib/html2rss/html/article_extractor/category_extractor.rb,
lib/html2rss/html/article_extractor/enclosure_extractor.rb,
lib/html2rss/selectors/post_processors/html_to_markdown.rb,
lib/html2rss/selectors/post_processors/markdown_to_html.rb,
lib/html2rss/auto_source/scraper/json_state/value_finder.rb,
lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb,
lib/html2rss/auto_source/scraper/schema/category_extractor.rb,
lib/html2rss/auto_source/scraper/json_state/document_scanner.rb,
lib/html2rss/request_service/puppet_commander/preload_runner.rb,
lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb,
lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb,
lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb,
lib/html2rss/request_service/puppet_commander/navigation_guards.rb,
lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb,
lib/html2rss/selectors/post_processors/html_transformers/wrap_img_in_a.rb,
lib/html2rss/auto_source/scraper/wordpress_api/page_scope/date_archive_range.rb,
lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb

Overview

rubocop:disable Metrics/ModuleLength -- Channel value object owns extract + freeze + Marshal

Defined Under Namespace

Modules: FeedBuilder, HashUtil, Html, LinkDestination, Log, SST, Scoring Classes: Article, AutoSource, CLI, Channel, Config, Defaults, Error, FeedPipeline, FeedResult, NoFeedItemsExtracted, RequestService, RequestSession, Selectors, Status, Url

Constant Summary collapse

VERSION = Current application version.

'0.25.0'

Class Method Summary collapse

.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ Hash
Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.
.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ RSS::Rss
Scrapes the provided URL without hand-written selectors and returns an RSS object.
.config_from_yaml_file(file, feed_name = nil) ⇒ Hash{Symbol => Object}
Loads a feed configuration from YAML.
.configure {|config| ... } ⇒ Html2rss::Defaults
Configures global library defaults.
.defaults ⇒ Html2rss::Defaults
The global defaults instance.
.feed(raw_config) ⇒ RSS::Rss
Returns an RSS object generated from the provided configuration.
.feed_result(raw_config) ⇒ Html2rss::FeedResult
Returns an opaque, Marshal-cacheable result of one scrape.
.json_feed(raw_config, feed_url: nil) ⇒ Hash
Returns a JSONFeed 1.1 hash generated from the provided configuration.
.logger ⇒ Object
The logger.
.logger=(logger) ⇒ Object

Class Method Details

.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ `Hash`

Scrapes the provided URL without hand-written selectors and returns a JSONFeed 1.1 hash.

Same auto_source pipeline as auto_source; see docs/auto_source.md.

Parameters:

url (String) —
source page URL
strategy (Symbol) (defaults to: :auto) —
request strategy to use
items_selector (String, nil) (defaults to: nil) —
optional selector hint for item extraction
max_redirects (Integer, nil) (defaults to: nil) —
optional redirect limit override
max_requests (Integer) (defaults to: 4) —
optional request budget override (default: 4 for sitemap sub-fetches)
local_file_path (String, nil) (defaults to: nil) —
optional local HTML file path
limit (Integer, nil) (defaults to: nil) —
max articles to keep (default: Html2rss::AutoSource::DEFAULT_LIMIT)

Returns:

(Hash) —
JSONFeed-compliant hash

# File 'lib/html2rss.rb', line 110

def self.auto_json_feed(url,
                        strategy: :auto,
                        items_selector: nil,
                        max_redirects: nil,
                        max_requests: 4,
                        local_file_path: nil,
                        limit: nil)
  json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
                                     local_file_path:, limit:))
end

.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ `RSS::Rss`

Scrapes the provided URL without hand-written selectors and returns an RSS object.

Builds an auto_source config, then FeedPipeline runs structured scrapers and (when needed) the SST heuristic path — see docs/auto_source.md.

Parameters:

url (String) —
source page URL
strategy (Symbol) (defaults to: :auto) —
request strategy to use
items_selector (String, nil) (defaults to: nil) —
optional selector hint for item extraction
max_redirects (Integer, nil) (defaults to: nil) —
optional redirect limit override
max_requests (Integer) (defaults to: 4) —
optional request budget override (default: 4 for sitemap sub-fetches)
local_file_path (String, nil) (defaults to: nil) —
optional local HTML file path
limit (Integer, nil) (defaults to: nil) —
max articles to keep (default: Html2rss::AutoSource::DEFAULT_LIMIT)

Returns:

(RSS::Rss) —
generated RSS feed

# File 'lib/html2rss.rb', line 86

def self.auto_source(url,
                     strategy: :auto,
                     items_selector: nil,
                     max_redirects: nil,
                     max_requests: 4,
                     local_file_path: nil,
                     limit: nil)
  feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
                                local_file_path:, limit:))
end

.config_from_yaml_file(file, feed_name = nil) ⇒ `Hash{Symbol => Object}`

Loads a feed configuration from YAML.

Parameters:

file (String) —
path to the YAML file
feed_name (String, nil) (defaults to: nil) —
optional feed name inside a multi-feed config

Returns:

(Hash{Symbol => Object}) —
loaded configuration hash



36
37
38

# File 'lib/html2rss.rb', line 36

def self.config_from_yaml_file(file, feed_name = nil)
  Config.load_yaml(file, feed_name)
end

.configure {|config| ... } ⇒ `Html2rss::Defaults`

Configures global library defaults.

Yield Parameters:

config (Html2rss::Defaults)

Returns:

(Html2rss::Defaults) —
the frozen defaults

# File 'lib/html2rss.rb', line 136

def configure
  config = defaults.dup
  yield config
  @defaults = config.freeze
end

.defaults ⇒ `Html2rss::Defaults`

Returns the global defaults instance.

Returns:

(Html2rss::Defaults) —
the global defaults instance



127
128
129

# File 'lib/html2rss.rb', line 127

def defaults
  @defaults ||= Defaults.new.freeze
end

.feed(raw_config) ⇒ `RSS::Rss`

Returns an RSS object generated from the provided configuration.

Parameters:

raw_config (Hash{Symbol => Object}) —
feed configuration

Returns:

(RSS::Rss) —
generated RSS feed



56
57
58

# File 'lib/html2rss.rb', line 56

def self.feed(raw_config)
  feed_result(raw_config).to_rss
end

.feed_result(raw_config) ⇒ `Html2rss::FeedResult`

Returns an opaque, Marshal-cacheable result of one scrape.

Prefer this when the same scrape must render as RSS and JSON Feed (e.g. web cache).

Parameters:

raw_config (Hash{Symbol => Object}) —
feed configuration

Returns:

(Html2rss::FeedResult)



47
48
49

# File 'lib/html2rss.rb', line 47

def self.feed_result(raw_config)
  FeedPipeline.new(raw_config).to_result
end

.json_feed(raw_config, feed_url: nil) ⇒ `Hash`

Returns a JSONFeed 1.1 hash generated from the provided configuration.

Parameters:

raw_config (Hash{Symbol => Object}) —
feed configuration
feed_url (String, nil) (defaults to: nil) —
optional self URL for the feed (JSON Feed feed_url)

Returns:

(Hash) —
JSONFeed-compliant hash



66
67
68

# File 'lib/html2rss.rb', line 66

def self.json_feed(raw_config, feed_url: nil)
  feed_result(raw_config).to_json_feed(feed_url:)
end

.logger ⇒ `Object`

Returns the logger.

Returns:

(Object) —
the logger



144
145
146

# File 'lib/html2rss.rb', line 144

def logger
  defaults.logger
end

.logger=(logger) ⇒ `Object`

Parameters:

logger (Object) —
the new logger



150
151
152

# File 'lib/html2rss.rb', line 150

def logger=(logger)
  configure { |config| config.logger = logger }
end

Module: Html2rss

Overview

Defined Under Namespace

Constant Summary collapse

Class Method Summary collapse

Class Method Details

.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ Hash

.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ RSS::Rss

.config_from_yaml_file(file, feed_name = nil) ⇒ Hash{Symbol => Object}

.configure {|config| ... } ⇒ Html2rss::Defaults

.defaults ⇒ Html2rss::Defaults

.feed(raw_config) ⇒ RSS::Rss

.feed_result(raw_config) ⇒ Html2rss::FeedResult

.json_feed(raw_config, feed_url: nil) ⇒ Hash

.logger ⇒ Object

.logger=(logger) ⇒ Object

.auto_json_feed(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ `Hash`

.auto_source(url, strategy: :auto, items_selector: nil, max_redirects: nil, max_requests: 4, local_file_path: nil, limit: nil) ⇒ `RSS::Rss`

.config_from_yaml_file(file, feed_name = nil) ⇒ `Hash{Symbol => Object}`

.configure {|config| ... } ⇒ `Html2rss::Defaults`

.defaults ⇒ `Html2rss::Defaults`

.feed(raw_config) ⇒ `RSS::Rss`

.feed_result(raw_config) ⇒ `Html2rss::FeedResult`

.json_feed(raw_config, feed_url: nil) ⇒ `Hash`

.logger ⇒ `Object`

.logger=(logger) ⇒ `Object`