Class: Scraper40

Inherits:
Scraper show all
Defined in:
lib/scraper.rb

Constant Summary collapse

RELEASE_NAME = 
'4.0'

Constants inherited from Scraper

Scraper::BASE_URL, Scraper::HEX_CHARS

Instance Attribute Summary

Attributes inherited from Scraper

#classes_for_method, #hrefs_for_name

Instance Method Summary collapse

Methods inherited from Scraper

#add_class, #add_file, #add_method, #get_home_page, #initialize, release_names, scrapers, #write_json

Constructor Details

This class inherits a constructor from Scraper

Instance Method Details

#add_classes(element) ⇒ Object 



183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/scraper.rb', line 183

def add_classes(element)
  REXML::XPath.each(element, "//*[@href]") do |element|
    class_name, class_href = add_class(element)
    url = File.join(BASE_URL, RELEASE_NAME, class_href)
    uri = URI(url)
    response = Net::HTTP.get_response(uri)
    raise response.code unless response.code == '200'
    response.body.lines.each do |line|
      next unless line.match(%r[<li ><a href="#method-])
      line.chomp!
      doc = REXML::Document.new(line)
      REXML::XPath.each(doc.root, "//*[@href]") do |element|
        add_method(element, class_name)
      end
    end
  end
end

#scrape ⇒ Object 



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/scraper.rb', line 161

def scrape
  home_page = get_home_page(RELEASE_NAME, '')
  home_page.lines.each do |line|
    line.chomp!
    next unless line.match(%r[<a href="])
    line += '</a>' unless line.match(%r[</a>]) # Add end-tag if needed.
    # Parse the line as XML.
    doc = REXML::Document.new(line)
    root = doc.root
    case root.name
    when 'a'  # Each file URL is the href attribute in an anchor element.
      add_file(root)
    when 'ul' # All class/module URLs are in a single ul element.
      add_classes(root)
      break # We don't need anything farther down.
    else
      # Ignore.
    end
  end
  write_json(RELEASE_NAME)
end