Class: Html2rss::AutoSource

Inherits:
Object
  • Object
show all
Defined in:
lib/html2rss/auto_source.rb,
lib/html2rss/auto_source/article.rb,
lib/html2rss/auto_source/channel.rb,
lib/html2rss/auto_source/cleanup.rb,
lib/html2rss/auto_source/reducer.rb,
lib/html2rss/auto_source/scraper.rb,
lib/html2rss/auto_source/rss_builder.rb,
lib/html2rss/auto_source/scraper/schema.rb,
lib/html2rss/auto_source/scraper/schema/base.rb,
lib/html2rss/auto_source/scraper/semantic_html.rb,
lib/html2rss/auto_source/scraper/semantic_html/image.rb,
lib/html2rss/auto_source/scraper/semantic_html/extractor.rb

Overview

The AutoSource class is responsible for extracting channel and articles from a given URL. It uses a set of ArticleExtractors to extract articles, utilizing popular ways of marking articles, e.g. schema, microdata, open graph, etc.

Defined Under Namespace

Modules: Scraper Classes: Article, Channel, Cleanup, NoArticlesFound, Reducer, RssBuilder, UnsupportedUrlScheme

Constant Summary collapse

SUPPORTED_URL_SCHEMES = 
%w[http https].to_set.freeze

Instance Method Summary collapse

Constructor Details

#initialize(url) ⇒ AutoSource

Returns a new instance of AutoSource.

Raises:

  • (ArgumentError) 


19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/html2rss/auto_source.rb', line 19

def initialize(url)
  unless url.is_a?(String) || url.is_a?(Addressable::URI)
    raise ArgumentError,
          'URL must be a String or Addressable::URI'
  end

  @url = Addressable::URI.parse(url)

  raise ArgumentError, 'URL must be absolute' unless @url.absolute?
  raise UnsupportedUrlScheme, "#{@url.scheme} not supported" unless SUPPORTED_URL_SCHEMES.include?(@url.scheme)
end

Instance Method Details

#articles ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/html2rss/auto_source.rb', line 43

def articles
  @articles ||= Scraper.from(parsed_body).flat_map do |scraper|
    instance = scraper.new(parsed_body, url:)

    articles_in_thread = Parallel.map(instance.each) do |article_hash|
      Log.debug "Scraper: #{scraper} in worker: #{Parallel.worker_number} [#{article_hash[:url]}]"

      Article.new(**article_hash, scraper:)
    end

    Reducer.call(articles_in_thread, url:)

    articles_in_thread
  end
end

#build ⇒ Object

Raises:



31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/html2rss/auto_source.rb', line 31

def build
  raise NoArticlesFound if articles.empty?

  Reducer.call(articles, url:)
  Cleanup.call(articles, url:, keep_different_domain: true)

  Html2rss::AutoSource::RssBuilder.new(
    channel:,
    articles:
  ).call
end

#channel ⇒ Object 



59
60
61
 
# File 'lib/html2rss/auto_source.rb', line 59

def channel
  Channel.new(parsed_body, response:, url:, articles:)
end