Class: OvertureMaps::Import::Downloader

Inherits:
Object
  • Object
show all
Defined in:
lib/overture_maps/import/downloader.rb

Overview

Downloads Overture data: whole theme files via anonymous HTTP, and bbox-filtered extracts via DuckDB (which pushes the bbox predicate down to parquet row-group statistics, so only the relevant slice of the dataset is transferred).

Constant Summary collapse

THEMES = 
%w[addresses base buildings divisions places transportation].freeze
TYPES = 
{
  "addresses" => %w[address],
  "base" => %w[bathymetry infrastructure land land_cover land_use water],
  "buildings" => %w[building building_part],
  "divisions" => %w[division division_area division_boundary],
  "places" => %w[place],
  "transportation" => %w[connector segment]
}.freeze
DIVISION_SUBTYPES =

division_area subtypes worth surfacing in a name search, roughly largest to smallest.

%w[country dependency region county localadmin locality borough neighborhood].freeze
MIN_SEARCH_AREA_KM2 =

Division areas smaller than this are sliver noise in a name search (ported from the parallel main-branch work, commit 6305670).

1.0
EXTRACT_FORMATS = 
{
  "parquet" => "parquet",
  "geojson" => "geojson",
  "geojsonseq" => "geojsonseq",
  "gpkg" => "gpkg",
  "geopackage" => "gpkg"
}.freeze

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(theme:, type: nil, release: nil, output_dir: nil) ⇒ Downloader

Returns a new instance of Downloader.

Raises:

  • (ArgumentError) 


41
42
43
44
45
46
47
48
49
 
# File 'lib/overture_maps/import/downloader.rb', line 41

def initialize(theme:, type: nil, release: nil, output_dir: nil)
  raise ArgumentError, "unknown theme: #{theme}" unless THEMES.include?(theme)
  raise ArgumentError, "unknown type #{type} for #{theme}" if type && !TYPES[theme].include?(type)

  @theme = theme
  @type = type
  @release = Releases.validate!(release || Releases.current)
  @output_dir = output_dir || OvertureMaps.configuration.cache_dir
end

Instance Attribute Details

#output_dir ⇒ Object (readonly)

Returns the value of attribute output_dir.



39
40
41
 
# File 'lib/overture_maps/import/downloader.rb', line 39

def output_dir
  @output_dir
end

#release ⇒ Object (readonly)

Returns the value of attribute release.



39
40
41
 
# File 'lib/overture_maps/import/downloader.rb', line 39

def release
  @release
end

#theme ⇒ Object (readonly)

Returns the value of attribute theme.



39
40
41
 
# File 'lib/overture_maps/import/downloader.rb', line 39

def theme
  @theme
end

#type ⇒ Object (readonly)

Returns the value of attribute type.



39
40
41
 
# File 'lib/overture_maps/import/downloader.rb', line 39

def type
  @type
end

Class Method Details

.bbox_query(theme:, type:, release:, bbox:, limit: nil) ⇒ Object

Builds the bbox-filtered SELECT for one theme/type. Intersection semantics: any feature whose bbox overlaps the query box is included, matching what "give me this area" means (the old strict-containment filter silently dropped everything touching the boundary).

Raises:

  • (ArgumentError) 


201
202
203
204
205
206
207
208
209
210
211
212
213
214
 
# File 'lib/overture_maps/import/downloader.rb', line 201

def self.bbox_query(theme:, type:, release:, bbox:, limit: nil)
  raise ArgumentError, "unknown theme: #{theme}" unless THEMES.include?(theme)
  raise ArgumentError, "unknown type #{type} for #{theme}" unless TYPES[theme].include?(type)

  source = source_glob(theme: theme, type: type, release: Releases.validate!(release))
  sql = <<~SQL
    SELECT *
    FROM read_parquet('#{source}', hive_partitioning=1)
    WHERE bbox.xmin <= ? AND bbox.xmax >= ?
      AND bbox.ymin <= ? AND bbox.ymax >= ?
  SQL
  sql += "LIMIT #{Integer(limit)}\n" if limit
  [sql, [bbox.max_lng, bbox.min_lng, bbox.max_lat, bbox.min_lat]]
end

.list_themes(release: nil) ⇒ Object 



152
153
154
155
156
 
# File 'lib/overture_maps/import/downloader.rb', line 152

def self.list_themes(release: nil)
  release = Releases.validate!(release || Releases.current)
  listing = Storage.list(prefix: "release/#{release}/", delimiter: "/")
  listing[:prefixes].filter_map { |p| p[/theme=([^\/]+)/, 1] }.sort
end

.list_types(theme:, release: nil) ⇒ Object 



146
147
148
149
150
 
# File 'lib/overture_maps/import/downloader.rb', line 146

def self.list_types(theme:, release: nil)
  release = Releases.validate!(release || Releases.current)
  listing = Storage.list(prefix: "release/#{release}/theme=#{theme}/", delimiter: "/")
  listing[:prefixes].filter_map { |p| p[/type=([^\/]+)/, 1] }.sort
end

.search_divisions(query:, release: nil, limit: 20) ⇒ Object

Searches division areas by name. Returns hashes with :id, :name, :subtype, :country, :region, :bbox (BoundingBox) and :area_km2, largest areas first.



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/overture_maps/import/downloader.rb', line 161

def self.search_divisions(query:, release: nil, limit: 20)
  release = Releases.validate!(release || Releases.current)
  source = source_glob(theme: "divisions", type: "division_area", release: release)
  placeholders = DIVISION_SUBTYPES.map { "?" }.join(", ")

  sql = <<~SQL
    SELECT id, names.primary AS name, subtype, country, region,
           bbox.xmin AS xmin, bbox.xmax AS xmax, bbox.ymin AS ymin, bbox.ymax AS ymax
    FROM read_parquet('#{source}', hive_partitioning=1)
    WHERE names.primary ILIKE ?
      AND subtype IN (#{placeholders})
      AND bbox.xmax > bbox.xmin AND bbox.ymax > bbox.ymin
    ORDER BY (bbox.xmax - bbox.xmin) * (bbox.ymax - bbox.ymin) DESC
    LIMIT #{Integer(limit)}
  SQL

  rows = QueryEngine.instance.query(sql, ["%#{query}%"] + DIVISION_SUBTYPES)
  results = rows.map do |row|
    bbox = BoundingBox.new(
      lat1: row["ymin"], lng1: row["xmin"],
      lat2: row["ymax"], lng2: row["xmax"],
      display_name: row["name"]
    )
    {
      id: row["id"],
      name: row["name"],
      subtype: row["subtype"],
      country: row["country"],
      region: row["region"],
      bbox: bbox,
      area_km2: Util.bbox_area_km2(bbox)
    }
  end
  results.select { |r| r[:area_km2] >= MIN_SEARCH_AREA_KM2 }
end

.source_glob(theme:, type:, release:) ⇒ Object 



216
217
218
 
# File 'lib/overture_maps/import/downloader.rb', line 216

def self.source_glob(theme:, type:, release:)
  "#{OvertureMaps.configuration.s3_uri.chomp("/")}/release/#{release}/theme=#{theme}/type=#{type}/*.parquet"
end

.themes_with_types ⇒ Object 



55
56
57
 
# File 'lib/overture_maps/import/downloader.rb', line 55

def self.themes_with_types
  TYPES
end

.types_for_theme(theme) ⇒ Object 



51
52
53
 
# File 'lib/overture_maps/import/downloader.rb', line 51

def self.types_for_theme(theme)
  TYPES[theme] || []
end

Instance Method Details

#cached_extract(bbox, format: "parquet") ⇒ Object

An existing extract for this theme/type/release/area, or nil. Matches exactly — never falls back to "some other file for the theme".



136
137
138
139
 
# File 'lib/overture_maps/import/downloader.rb', line 136

def cached_extract(bbox, format: "parquet")
  path = extract_path(bbox, format: format)
  File.exist?(path) && File.size(path).positive? ? path : nil
end

#download_theme_files ⇒ Object

Downloads the complete parquet files for this theme/type. These are large (buildings alone is hundreds of GB globally) — bbox extracts are almost always what you want instead.



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/overture_maps/import/downloader.rb', line 62

def download_theme_files
  files = list_files
  if files.empty?
    log "No files found for #{theme}#{type ? "/#{type}" : ""} in #{release}"
    return 0
  end

  log "Found #{files.count} file(s) to download..."
  FileUtils.mkdir_p(output_dir)

  files.each do |file|
    filename = File.basename(file[:key])
    local_path = File.join(output_dir, filename)

    result = Storage.download(file[:key], to: local_path, expected_size: file[:size])
    if result == :skipped
      log "Skipping #{filename} (already exists)"
    else
      log "Downloaded #{filename} (#{Util.format_size(file[:size])})"
    end
  end

  files.count
end

#extract_bbox(bbox, format: "parquet", output_path: nil, limit: nil) ⇒ Object

Writes a bbox-filtered extract for one type to a local file and returns its path (nil when the area has no data). The extract is named by theme/type/release/area, so later runs reuse it as a cache.



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/overture_maps/import/downloader.rb', line 90

def extract_bbox(bbox, format: "parquet", output_path: nil, limit: nil)
  target_type = type or raise ArgumentError, "extract_bbox requires a type"
  path = output_path || extract_path(bbox, format: format)
  FileUtils.mkdir_p(File.dirname(path))

  sql, params = self.class.bbox_query(theme: theme, type: target_type, release: release,
                                      bbox: bbox, limit: limit)
  begin
    QueryEngine.instance.copy_to(sql, params: params, output_path: path,
                                 format: EXTRACT_FORMATS.fetch(format.to_s.downcase, "parquet"))
  rescue QueryEngine::Error, StandardError => e
    raise release_hint_error(e)
  end

  if File.exist?(path) && File.size(path).positive?
    path
  else
    FileUtils.rm_f(path)
    nil
  end
end

#extract_bbox_all_types(bbox, format: "parquet") ⇒ Object

Extracts every type in the theme for a bbox. Returns the paths written.



113
114
115
116
117
118
119
120
121
122
 
# File 'lib/overture_maps/import/downloader.rb', line 113

def extract_bbox_all_types(bbox, format: "parquet")
  types = type ? [type] : TYPES.fetch(theme)
  types.filter_map do |t|
    log "Querying #{theme}/#{t} (#{release})..."
    path = self.class.new(theme: theme, type: t, release: release, output_dir: output_dir)
               .extract_bbox(bbox, format: format)
    log(path ? "  Saved #{File.basename(path)} (#{Util.format_size(File.size(path))})" : "  No data found")
    path
  end
end

#extract_nearby(lat:, lng:, radius_meters:, format: "parquet") ⇒ Object 



124
125
126
 
# File 'lib/overture_maps/import/downloader.rb', line 124

def extract_nearby(lat:, lng:, radius_meters:, format: "parquet")
  extract_bbox_all_types(BoundingBox.around(lat: lat, lng: lng, radius_meters: radius_meters), format: format)
end

#extract_path(bbox, format: "parquet") ⇒ Object

The cache path for a bbox extract of this theme/type/release.



129
130
131
132
 
# File 'lib/overture_maps/import/downloader.rb', line 129

def extract_path(bbox, format: "parquet")
  ext = EXTRACT_FORMATS.fetch(format.to_s.downcase, "parquet")
  File.join(output_dir, "#{theme}_#{type}_#{release}_#{bbox.slug}.#{ext}")
end

#list_files ⇒ Object 



141
142
143
144
 
# File 'lib/overture_maps/import/downloader.rb', line 141

def list_files
  prefix = "release/#{release}/theme=#{theme}#{type ? "/type=#{type}" : ""}"
  Storage.list(prefix: prefix)[:objects].select { |o| o[:key].end_with?(".parquet") }
end