Class: FetchUtil::Fetcher::PayloadSnapshot

Inherits:
Object
  • Object
show all
Defined in:
lib/fetch_util/fetcher.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(payload:, requested_url:, final_url:, canonical_url:, raw_final_url:, raw_canonical_url:) ⇒ PayloadSnapshot

Returns a new instance of PayloadSnapshot.



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/fetch_util/fetcher.rb', line 93

def initialize(payload:, requested_url:, final_url:, canonical_url:, raw_final_url:, raw_canonical_url:)
  @payload = payload
  @requested_url = requested_url
  @final_url = final_url
  @canonical_url = canonical_url
  @raw_final_url = raw_final_url
  @raw_canonical_url = raw_canonical_url
  @markdown = payload["markdown"].to_s
  @normalized_markdown = FetchUtil.normalize_whitespace(@markdown)
  @content_downcase = FetchUtil.normalize_whitespace([payload["title"], @markdown].compact.join(" ")).downcase
  @context = FetchUtil.normalize_whitespace([payload["title"], payload["siteName"], @markdown].join(" "))
  @context_downcase = @context.downcase
  @context_with_excerpt_downcase = FetchUtil.normalize_whitespace(
    [payload["title"], @markdown, payload["excerpt"]].compact.join(" ")
  ).downcase
  @first_20_context = first_lines_context(20)
  @first_80_context = first_lines_context(80)
  @plain_list_item_count = @markdown.lines.grep(/^\s*(?:\d+\.\s+|[-*]\s+)/).count
  @linked_heading_count = @markdown.scan(Fetcher::LINKED_MARKDOWN_HEADING_PATTERN).count
  @linked_item_count = @markdown.scan(Fetcher::LINKED_MARKDOWN_ITEM_PATTERN).count
  @markdown_link_count = @markdown.scan(%r{\]\(https?://}).count
  @prose_lines = @markdown.lines.reject { |line| line.match?(/^\s*(?:#|[-*]\s+|\d+\.\s+)/) }
  @long_prose_line_count_one_hundred = long_prose_line_count(100)
  @long_prose_line_count_one_twenty = long_prose_line_count(120)
  @long_prose_line_count_one_forty = long_prose_line_count(140)
  @list_link_count = @markdown.lines.grep(/^\s*- \[/).count
  @table_row_count = @markdown.lines.grep(/^\|/).count
  @heading_count = @markdown.lines.grep(/^(?:#){1,3}\s+/).count
  @common_tokens = @context_downcase.scan(/[a-z0-9]{3,}/).uniq
  @resource_urls = [requested_url, final_url, canonical_url].compact
  @raw_resource_urls = [requested_url, raw_final_url, raw_canonical_url].compact
  @resource_url_facets = @resource_urls.to_h { |url| [url, url_facets(url)] }
end

Instance Attribute Details

#canonical_url ⇒ Object (readonly)

Returns the value of attribute canonical_url.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def canonical_url
  @canonical_url
end

#common_tokens ⇒ Object (readonly)

Returns the value of attribute common_tokens.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def common_tokens
  @common_tokens
end

#content_downcase ⇒ Object (readonly)

Returns the value of attribute content_downcase.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def content_downcase
  @content_downcase
end

#context ⇒ Object (readonly)

Returns the value of attribute context.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def context
  @context
end

#context_downcase ⇒ Object (readonly)

Returns the value of attribute context_downcase.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def context_downcase
  @context_downcase
end

#context_with_excerpt_downcase ⇒ Object (readonly)

Returns the value of attribute context_with_excerpt_downcase.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def context_with_excerpt_downcase
  @context_with_excerpt_downcase
end

#final_url ⇒ Object (readonly)

Returns the value of attribute final_url.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def final_url
  @final_url
end

#first_20_context ⇒ Object (readonly)

Returns the value of attribute first_20_context.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def first_20_context
  @first_20_context
end

#first_80_context ⇒ Object (readonly)

Returns the value of attribute first_80_context.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def first_80_context
  @first_80_context
end

#heading_count ⇒ Object (readonly)

Returns the value of attribute heading_count.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def heading_count
  @heading_count
end

#linked_heading_count ⇒ Object (readonly)

Returns the value of attribute linked_heading_count.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def linked_heading_count
  @linked_heading_count
end

#linked_item_count ⇒ Object (readonly)

Returns the value of attribute linked_item_count.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def linked_item_count
  @linked_item_count
end

#list_link_count ⇒ Object (readonly)

Returns the value of attribute list_link_count.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def list_link_count
  @list_link_count
end

#long_prose_line_count_one_forty ⇒ Object (readonly)

Returns the value of attribute long_prose_line_count_one_forty.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def long_prose_line_count_one_forty
  @long_prose_line_count_one_forty
end

#long_prose_line_count_one_hundred ⇒ Object (readonly)

Returns the value of attribute long_prose_line_count_one_hundred.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def long_prose_line_count_one_hundred
  @long_prose_line_count_one_hundred
end

#long_prose_line_count_one_twenty ⇒ Object (readonly)

Returns the value of attribute long_prose_line_count_one_twenty.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def long_prose_line_count_one_twenty
  @long_prose_line_count_one_twenty
end

#markdown ⇒ Object (readonly)

Returns the value of attribute markdown.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def markdown
  @markdown
end

#markdown_link_count ⇒ Object (readonly)

Returns the value of attribute markdown_link_count.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def markdown_link_count
  @markdown_link_count
end

#normalized_markdown ⇒ Object (readonly)

Returns the value of attribute normalized_markdown.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def normalized_markdown
  @normalized_markdown
end

#payload ⇒ Object (readonly)

Returns the value of attribute payload.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def payload
  @payload
end

#plain_list_item_count ⇒ Object (readonly)

Returns the value of attribute plain_list_item_count.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def plain_list_item_count
  @plain_list_item_count
end

#prose_lines ⇒ Object (readonly)

Returns the value of attribute prose_lines.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def prose_lines
  @prose_lines
end

#raw_canonical_url ⇒ Object (readonly)

Returns the value of attribute raw_canonical_url.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def raw_canonical_url
  @raw_canonical_url
end

#raw_final_url ⇒ Object (readonly)

Returns the value of attribute raw_final_url.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def raw_final_url
  @raw_final_url
end

#raw_resource_urls ⇒ Object (readonly)

Returns the value of attribute raw_resource_urls.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def raw_resource_urls
  @raw_resource_urls
end

#requested_url ⇒ Object (readonly)

Returns the value of attribute requested_url.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def requested_url
  @requested_url
end

#resource_url_facets ⇒ Object (readonly)

Returns the value of attribute resource_url_facets.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def resource_url_facets
  @resource_url_facets
end

#resource_urls ⇒ Object (readonly)

Returns the value of attribute resource_urls.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def resource_urls
  @resource_urls
end

#table_row_count ⇒ Object (readonly)

Returns the value of attribute table_row_count.



84
85
86
 
# File 'lib/fetch_util/fetcher.rb', line 84

def table_row_count
  @table_row_count
end

Instance Method Details

#path_key(url) ⇒ Object 



132
133
134
 
# File 'lib/fetch_util/fetcher.rb', line 132

def path_key(url)
  resource_url_facets.fetch(url) { url_facets(url) }[:path_key]
end

#search_or_list_resource_url?(url) ⇒ Boolean

Returns:

  • (Boolean) 


127
128
129
130
 
# File 'lib/fetch_util/fetcher.rb', line 127

def search_or_list_resource_url?(url)
  facet = resource_url_facets.fetch(url) { url_facets(url) }
  facet[:index_query] || facet[:search_or_list_path]
end