Class: Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/scraper.rb

Overview

Class to scrape Ruby info from the documentation site and store it as JSON.

Direct Known Subclasses

Scraper32, Scraper34, Scraper40

Constant Summary collapse

BASE_URL =

The official documentation site for English.

'https://docs.ruby-lang.org/en'
HEX_CHARS =

Translations for hex characters.

{
  '-21' => '!',
  '-25' => '%',
  '-26' => '&',
  '-2A' => '*',
  '-2B' => '+',
  '-2D' => '-',
  '-2F' => '/',
  '-3C' => '<',
  '-40' => '@',
  '-3D' => '=',
  '-3E' => '>',
  '-3F' => '?',
  '-5B' => '[',
  '-5D' => ']',
  '-5E' => '^',
  '-60' => '`',
  '-7C' => '|',
}

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Scraper

Returns a new instance of Scraper.



41
42
43
44
 
# File 'lib/scraper.rb', line 41

def initialize
  self.hrefs_for_name = {}
  self.classes_for_method = {}
end

Instance Attribute Details

#classes_for_method ⇒ Object

Hash: the names of the classes that have a method.



39
40
41
 
# File 'lib/scraper.rb', line 39

def classes_for_method
  @classes_for_method
end

#hrefs_for_name ⇒ Object

Hash: relative URL paths for a name.



36
37
38
 
# File 'lib/scraper.rb', line 36

def hrefs_for_name
  @hrefs_for_name
end

Class Method Details

.release_names ⇒ Object 



54
55
56
 
# File 'lib/scraper.rb', line 54

def self.release_names
  self.scrapers.keys
end

.scrapers ⇒ Object 



46
47
48
49
50
51
52
 
# File 'lib/scraper.rb', line 46

def self.scrapers
  {
    '3.2' => Scraper34,
    '3.4' => Scraper32,
    '4.0' => Scraper40,
  }
end

Instance Method Details

#add_class(element) ⇒ Object

Argument is an REXML::Element representing a link to a class. Returns the parsed name and href.



86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/scraper.rb', line 86

def add_class(element)
  unless element.name == 'a'
    message = "Expecting 'a', not '#{element.name}'"
    raise ArgumentError.new(message)
  end
  class_href = element.attributes['href'].sub(%r[^./], '')
  class_name = class_href.gsub('/', '::').sub(%r[\.html$], '')
  puts "Adding class #{class_name}"
  hrefs_for_name[class_name] = [class_href]
  [class_name, class_href]
end

#add_file(element) ⇒ Object

Argument is an REXML::Element representing a link to a file. Returns the parsed name and href.



71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/scraper.rb', line 71

def add_file(element)
  unless element.name == 'a'
    message = "Expecting 'a', not '#{element.name}'"
    raise ArgumentError.new(message)
  end
  file_href = element.attributes['href']
  file_href.sub!(%r[^./], '')
  file_name = 'ruby:' + file_href.sub(/\.html$/, '')
  puts "Adding file #{file_name}"
  hrefs_for_name[file_name] = [file_href]
  [file_name, file_href]
end

#add_method(element, class_name) ⇒ Object

Arguments are an REXML::Element representing a link to a class, and the class name. Returns the parsed name and href.



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/scraper.rb', line 100

def add_method(element, class_name)
  method_href = element.attributes['href']
  # Translate hex characters.
  method_name = method_href.dup
  hex_chars = method_name.scan(/-[0-9A-F]{2}/)
  hex_chars.each do |hex_char|
    char = HEX_CHARS[hex_char]
    raise hex_char unless char
    method_name.sub!(hex_char, char)
  end
  method_name = case
                # when method_href == '#method-i-2D'
                #   # Special case; not worth the trouble of handling below.
                #   '#-'
                when method_href.match(/[#]?method-i-2D/)
                  # Special case; not worth the trouble of handling below.
                  '#-'
                # when method_href == '#method-i-2D-40'
                #   # Special case; not worth the trouble of handling below.
                #   '#-@'
                when method_href.match(/[#]?method-i-2D-40/)
                  # Special case; not worth the trouble of handling below.
                  '#-@'
                when method_href.match('-i-')
                  # Instance method.
                  '#' + method_name.sub(/[#]?method-i[-]?/, '')
                when method_href.match('-c-')
                  # Singleton method.
                  '::' + method_name.sub(/[#]?method-c[-]?/, '')
                end
  puts "  Adding method #{method_name}"
  hrefs_for_name[method_name] = "#{method_href}"
  classes_for_method[method_name] ||= []
  classes_for_method[method_name] << "#{class_name}"
  [method_name, method_href]
end

#get_home_page(release_name, suffix) ⇒ Object 



58
59
60
61
62
63
64
65
66
67
 
# File 'lib/scraper.rb', line 58

def get_home_page(release_name, suffix)
  url = File.join(BASE_URL, release_name, suffix)
  uri = URI(url)
  response = Net::HTTP.get_response(uri)
  unless response.code == '200'
    message = "Page #{url} for release #{url} not found; code #{response.code}."
    raise RuntimeError.new(message)
  end
  response.body
end

#write_json(release_name) ⇒ Object 



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
 
# File 'lib/scraper.rb', line 137

def write_json(release_name)
  # This JSON is bulky, but readable by humans.
  data = {
    :timestamp => Time.now,
    :hrefs_for_name => hrefs_for_name.sort.to_h,
    :classes_for_method => classes_for_method.sort.to_h,
  }
  json = JSON.generate(
    data.sort.to_h,
    indent: "  ",     # 4 spaces per level
    space: " ",       # space after :
    array_nl: "\n",   # newline after each array element
    object_nl: "\n"   # newline after each object member
  )
  filepath = "data/#{release_name}.json"
  File.write(filepath, json)
end