Class: Html2rss::AutoSource::Scraper::SemanticHtml::EntryDeduplicator

Inherits:
Object
  • Object
show all
Defined in:
lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb

Overview

Collapses nested containers and deduplicates ranked segments pointing to the same destination.

Instance Method Summary collapse

Constructor Details

#initialize(url, scraper: nil, link_resolver: nil) ⇒ EntryDeduplicator

Returns a new instance of EntryDeduplicator.

Parameters:

  • url (String, Html2rss::Url) —

    base url used to resolve relative hrefs

  • scraper (Class, nil) (defaults to: nil) —

    scraper class stamped onto extracted articles

  • link_resolver (Scoring::LinkResolver, nil) (defaults to: nil) —

    page-scoped destination facts cache



13
14
15
16
17
18
 
# File 'lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb', line 13

def initialize(url, scraper: nil, link_resolver: nil)
  @url = url
  @scraper = scraper
  @link_resolver = link_resolver || Scoring::LinkResolver.new(url)
  @article_cache = {}.compare_by_identity
end

Instance Method Details

#article_for(entry) ⇒ Html2rss::Article?

Parameters:

Returns:



34
35
36
37
38
39
40
41
42
43
 
# File 'lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb', line 34

def article_for(entry)
  @article_cache.fetch(entry) do
    @article_cache[entry] = ::Html2rss::Html::SstArticleExtractor.call(
      entry,
      base_url: @url,
      scraper: @scraper,
      fallback_anchorless: true
    )
  end
end

#call(ranked) ⇒ Array<Scoring::RankedSegment>

Parameters:

Returns:



23
24
25
26
27
28
29
 
# File 'lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb', line 23

def call(ranked)
  destination_groups(ranked).filter_map do |group|
    group.reduce do |best, entry|
      stronger_entry?(entry, best) ? entry : best
    end
  end
end

#stronger_entry?(left, right) ⇒ Boolean

Parameters:

Returns:

  • (Boolean) 


49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb', line 49

def stronger_entry?(left, right) # rubocop:disable Metrics/AbcSize
  final_delta = left.final_score <=> right.final_score
  return final_delta.positive? unless final_delta.zero?

  quality_delta = left.quality_score <=> right.quality_score
  return quality_delta.positive? unless quality_delta.zero?

  left_article = article_for(left)
  right_article = article_for(right)
  return !right_article if left_article.nil? || right_article.nil?

  richness_delta = payload_richness_signature(left_article) <=> payload_richness_signature(right_article)
  richness_delta.zero? ? left.position < right.position : richness_delta.positive?
end