Class: Aircana::Contexts::Web

Inherits:
Object
  • Object
show all
Includes:
HTTParty
Defined in:
lib/aircana/contexts/web.rb

Overview

rubocop:disable Metrics/ClassLength

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Web

Returns a new instance of Web.



20
21
22
 
# File 'lib/aircana/contexts/web.rb', line 20

def initialize
  @local_storage = Local.new
end

Instance Method Details

#fetch_url_for(kb_name:, url:, existing_metadata: nil) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/aircana/contexts/web.rb', line 24

def fetch_url_for(kb_name:, url:, existing_metadata: nil)
  validate_url!(url)

  page_data = fetch_and_process_url(url)
  store_page_as_markdown(page_data, kb_name)

  build_url_metadata(page_data, kb_name: kb_name, existing_metadata: existing_metadata)
rescue StandardError => e
  handle_fetch_error(url, e)
  nil
end

#fetch_urls_for(kb_name:, urls:, existing_metadata: nil) ⇒ Object

rubocop:disable Metrics/MethodLength



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/aircana/contexts/web.rb', line 36

def fetch_urls_for(kb_name:, urls:, existing_metadata: nil) # rubocop:disable Metrics/MethodLength
  return { pages_count: 0, sources: [] } if urls.empty?

  existing_metadata ||= load_existing_url_metadata(kb_name)
  pages_metadata = []
  successful_urls = []

  ProgressTracker.with_batch_progress(urls, "Fetching URLs") do |url, _index|
    metadata = fetch_url_for(kb_name: kb_name, url: url, existing_metadata: existing_metadata)
    if metadata
      pages_metadata << metadata
      successful_urls << url
    end
  end

  if successful_urls.any?
    sources = build_sources_metadata(successful_urls, pages_metadata)
    update_or_create_manifest(kb_name, sources)
    { pages_count: successful_urls.size, sources: sources }
  else
    { pages_count: 0, sources: [] }
  end
end

#refresh_web_sources(kb_name:) ⇒ Object

rubocop:disable Metrics/CyclomaticComplexity



60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/aircana/contexts/web.rb', line 60

def refresh_web_sources(kb_name:) # rubocop:disable Metrics/CyclomaticComplexity
  sources = Manifest.sources_from_manifest(kb_name)
  web_sources = sources.select { |s| s["type"] == "web" }

  return { pages_count: 0, sources: [] } if web_sources.empty?

  all_urls = web_sources.flat_map { |source| source["urls"]&.map { |u| u["url"] } || [] }
  return { pages_count: 0, sources: [] } if all_urls.empty?

  existing_metadata = load_existing_url_metadata(kb_name)
  fetch_urls_for(kb_name: kb_name, urls: all_urls, existing_metadata: existing_metadata)
end