Class: FetchUtil::Fetcher::PayloadSnapshot

Inherits:
Object
  • Object
show all
Defined in:
lib/fetch_util/fetcher.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(payload:, requested_url:, final_url:, canonical_url:, raw_final_url:, raw_canonical_url:) ⇒ PayloadSnapshot

Returns a new instance of PayloadSnapshot.



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/fetch_util/fetcher.rb', line 90

def initialize(payload:, requested_url:, final_url:, canonical_url:, raw_final_url:, raw_canonical_url:)
  @payload = payload
  @requested_url = requested_url
  @final_url = final_url
  @canonical_url = canonical_url
  @raw_final_url = raw_final_url
  @raw_canonical_url = raw_canonical_url
  @markdown = payload["markdown"].to_s
  @normalized_markdown = FetchUtil.normalize_whitespace(@markdown)
  @content_downcase = FetchUtil.normalize_whitespace([payload["title"], @markdown].compact.join(" ")).downcase
  @context = FetchUtil.normalize_whitespace([payload["title"], payload["siteName"], @markdown].join(" "))
  @context_downcase = @context.downcase
  @context_with_excerpt_downcase = FetchUtil.normalize_whitespace(
    [payload["title"], @markdown, payload["excerpt"]].compact.join(" ")
  ).downcase
  @first_20_context = first_lines_context(20)
  @first_80_context = first_lines_context(80)
  @plain_list_item_count = @markdown.lines.grep(/^\s*(?:\d+\.\s+|[-*]\s+)/).count
  @linked_heading_count = @markdown.scan(Fetcher::LINKED_MARKDOWN_HEADING_PATTERN).count
  @linked_item_count = @markdown.scan(Fetcher::LINKED_MARKDOWN_ITEM_PATTERN).count
  @markdown_link_count = @markdown.scan(%r{\]\(https?://}).count
  @prose_lines = @markdown.lines.reject { |line| line.match?(/^\s*(?:#|[-*]\s+|\d+\.\s+)/) }
  @long_prose_line_count_one_hundred = long_prose_line_count(100)
  @long_prose_line_count_one_twenty = long_prose_line_count(120)
  @long_prose_line_count_one_forty = long_prose_line_count(140)
  @list_link_count = @markdown.lines.grep(/^\s*- \[/).count
  @table_row_count = @markdown.lines.grep(/^\|/).count
  @heading_count = @markdown.lines.grep(/^(?:#){1,3}\s+/).count
  @common_tokens = @context_downcase.scan(/[a-z0-9]{3,}/).uniq
  @resource_urls = [requested_url, final_url, canonical_url].compact
  @raw_resource_urls = [requested_url, raw_final_url, raw_canonical_url].compact
  @resource_url_facets = @resource_urls.to_h { |url| [url, url_facets(url)] }
end

Instance Attribute Details

#canonical_url ⇒ Object (readonly)

Returns the value of attribute canonical_url.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def canonical_url
  @canonical_url
end

#common_tokens ⇒ Object (readonly)

Returns the value of attribute common_tokens.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def common_tokens
  @common_tokens
end

#content_downcase ⇒ Object (readonly)

Returns the value of attribute content_downcase.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def content_downcase
  @content_downcase
end

#context ⇒ Object (readonly)

Returns the value of attribute context.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def context
  @context
end

#context_downcase ⇒ Object (readonly)

Returns the value of attribute context_downcase.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def context_downcase
  @context_downcase
end

#context_with_excerpt_downcase ⇒ Object (readonly)

Returns the value of attribute context_with_excerpt_downcase.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def context_with_excerpt_downcase
  @context_with_excerpt_downcase
end

#final_url ⇒ Object (readonly)

Returns the value of attribute final_url.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def final_url
  @final_url
end

#first_20_context ⇒ Object (readonly)

Returns the value of attribute first_20_context.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def first_20_context
  @first_20_context
end

#first_80_context ⇒ Object (readonly)

Returns the value of attribute first_80_context.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def first_80_context
  @first_80_context
end

#heading_count ⇒ Object (readonly)

Returns the value of attribute heading_count.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def heading_count
  @heading_count
end

#linked_heading_count ⇒ Object (readonly)

Returns the value of attribute linked_heading_count.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def linked_heading_count
  @linked_heading_count
end

#linked_item_count ⇒ Object (readonly)

Returns the value of attribute linked_item_count.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def linked_item_count
  @linked_item_count
end

#list_link_count ⇒ Object (readonly)

Returns the value of attribute list_link_count.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def list_link_count
  @list_link_count
end

#long_prose_line_count_one_forty ⇒ Object (readonly)

Returns the value of attribute long_prose_line_count_one_forty.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def long_prose_line_count_one_forty
  @long_prose_line_count_one_forty
end

#long_prose_line_count_one_hundred ⇒ Object (readonly)

Returns the value of attribute long_prose_line_count_one_hundred.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def long_prose_line_count_one_hundred
  @long_prose_line_count_one_hundred
end

#long_prose_line_count_one_twenty ⇒ Object (readonly)

Returns the value of attribute long_prose_line_count_one_twenty.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def long_prose_line_count_one_twenty
  @long_prose_line_count_one_twenty
end

#markdown ⇒ Object (readonly)

Returns the value of attribute markdown.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def markdown
  @markdown
end

#markdown_link_count ⇒ Object (readonly)

Returns the value of attribute markdown_link_count.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def markdown_link_count
  @markdown_link_count
end

#normalized_markdown ⇒ Object (readonly)

Returns the value of attribute normalized_markdown.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def normalized_markdown
  @normalized_markdown
end

#payload ⇒ Object (readonly)

Returns the value of attribute payload.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def payload
  @payload
end

#plain_list_item_count ⇒ Object (readonly)

Returns the value of attribute plain_list_item_count.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def plain_list_item_count
  @plain_list_item_count
end

#prose_lines ⇒ Object (readonly)

Returns the value of attribute prose_lines.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def prose_lines
  @prose_lines
end

#raw_canonical_url ⇒ Object (readonly)

Returns the value of attribute raw_canonical_url.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def raw_canonical_url
  @raw_canonical_url
end

#raw_final_url ⇒ Object (readonly)

Returns the value of attribute raw_final_url.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def raw_final_url
  @raw_final_url
end

#raw_resource_urls ⇒ Object (readonly)

Returns the value of attribute raw_resource_urls.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def raw_resource_urls
  @raw_resource_urls
end

#requested_url ⇒ Object (readonly)

Returns the value of attribute requested_url.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def requested_url
  @requested_url
end

#resource_url_facets ⇒ Object (readonly)

Returns the value of attribute resource_url_facets.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def resource_url_facets
  @resource_url_facets
end

#resource_urls ⇒ Object (readonly)

Returns the value of attribute resource_urls.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def resource_urls
  @resource_urls
end

#table_row_count ⇒ Object (readonly)

Returns the value of attribute table_row_count.



81
82
83
 
# File 'lib/fetch_util/fetcher.rb', line 81

def table_row_count
  @table_row_count
end

Instance Method Details

#path_key(url) ⇒ Object 



129
130
131
 
# File 'lib/fetch_util/fetcher.rb', line 129

def path_key(url)
  resource_url_facets.fetch(url) { url_facets(url) }[:path_key]
end

#search_or_list_resource_url?(url) ⇒ Boolean

Returns:

  • (Boolean) 


124
125
126
127
 
# File 'lib/fetch_util/fetcher.rb', line 124

def search_or_list_resource_url?(url)
  facet = resource_url_facets.fetch(url) { url_facets(url) }
  facet[:index_query] || facet[:search_or_list_path]
end