Class: Detergent::Cleaner

Inherits:
Object
  • Object
show all
Defined in:
lib/detergent/cleaner.rb

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Cleaner

Returns a new instance of Cleaner.



4
5
6
7
8
 
# File 'lib/detergent/cleaner.rb', line 4

def initialize
  @node_scorer = NodeScorer.new
  @obvious_junk_matcher = Matchers::ObviousJunkMatcher.new
  @removable_node_matcher = Matchers::RemovableNodeMatcher.new
end

Instance Method Details

#clean(html) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/detergent/cleaner.rb', line 10

def clean(html)
  @html = html
  title, body = cleaned_html(@html)

  <<-HTML
  <html>
    <head>
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width, initial-scale=1">
      <title>#{title}</title>
    </head>
    #{body}
  </html>
  HTML
end

#cleaned_html(html) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/detergent/cleaner.rb', line 36

def cleaned_html(html)
  doc = Nokogiri::HTML(html)

  title_node = doc.at('title')
  title = title_node ? title_node.text.strip : ""

  body = doc.at('body')
  content = nil

  if body
    # First remove the most egregious crap, like obvious ads, etc.
    prune(node: body, matcher: Matchers::ObviousJunkMatcher.new)

    # Find the highest-scoring node in the cleaned tree
    highest_scoring = find_highest_scoring_node(body)
    return nil unless highest_scoring

    # Find the appropriate root container for that node
    content_root = find_content_root(highest_scoring)

    # Apply second-pass cleaning to the content
    if content_root
      clean_node(content_root)
    end

    content = content_root
  end

  [title, content]
end

#title(html) ⇒ Object 



26
27
28
29
30
31
32
33
 
# File 'lib/detergent/cleaner.rb', line 26

def title(html)
  doc = Nokogiri::HTML(html)

  title_node = doc.at('title')
  title = title_node ? title_node.text.strip : ""

  title
end