Class: Relaton::Itu::DataFetcher

Inherits:
Core::DataFetcher show all
Defined in:
lib/relaton/itu/data_fetcher.rb

Constant Summary collapse

SEARCH_URL = 
"https://www.itu.int/net4/ITU-T/search/GlobalSearch/RunSearch".freeze
ROWS = 
100
MAX_EMPTY_PAGES = 
3

Instance Method Summary collapse

Methods inherited from Core::DataFetcher

fetch, #gh_issue, #gh_issue_channel, #initialize, #output_file, #report_errors, #serialize

Constructor Details

This class inherits a constructor from Relaton::Core::DataFetcher

Instance Method Details

#fetch(_source = nil) ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/relaton/itu/data_fetcher.rb', line 22

def fetch(_source = nil)
  start = 0
  empty_pages = 0
  loop do
    results = search_request(start)
    if results.empty?
      empty_pages += 1
      break if empty_pages >= MAX_EMPTY_PAGES

      start += ROWS
      next
    end

    empty_pages = 0
    results.each do |result|
      bib = DataParserR.parse(result, @errors)
      write_file(bib) if bib
    rescue => e # rubocop:disable Style/RescueStandardError
      Util.error "#{e.message}\n#{e.backtrace}"
    end

    start += ROWS
  end
  index.save
  report_errors
end

#index ⇒ Object 



14
15
16
 
# File 'lib/relaton/itu/data_fetcher.rb', line 14

def index
  @index ||= Relaton::Index.find_or_create :itu, file: "index-v1.yaml"
end

#log_error(msg) ⇒ Object 



18
19
20
 
# File 'lib/relaton/itu/data_fetcher.rb', line 18

def log_error(msg)
  Util.error msg
end

#to_bibxml(bib) ⇒ Object 



70
71
72
 
# File 'lib/relaton/itu/data_fetcher.rb', line 70

def to_bibxml(bib)
  bib.to_rfcxml
end

#to_xml(bib) ⇒ Object 



66
67
68
 
# File 'lib/relaton/itu/data_fetcher.rb', line 66

def to_xml(bib)
  bib.to_xml bibdata: true
end

#to_yaml(bib) ⇒ Object 



62
63
64
 
# File 'lib/relaton/itu/data_fetcher.rb', line 62

def to_yaml(bib)
  bib.to_yaml
end

#write_file(bib) ⇒ Object

Parameters:



50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/relaton/itu/data_fetcher.rb', line 50

def write_file(bib) # rubocop:disable Metrics/AbcSize
  id = bib.docidentifier.find(&:primary).content
  file = output_file(id)
  if @files.include? file
    Util.warn "File #{file} exists."
  else
    @files << file
  end
  index.add_or_update id, file
  File.write file, serialize(bib), encoding: "UTF-8"
end