Class: OvertureMaps::Import::Runner

Inherits:
Object
  • Object
show all
Defined in:
lib/overture_maps/import/runner.rb

Overview

Batches records into idempotent upserts. Re-running an import updates existing rows (keyed on the GERS id primary key) instead of failing.

Constant Summary collapse

MAX_STORED_ERRORS = 
50
PROGRESS_INTERVAL = 
10_000

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(model_class:, theme: nil, type: nil, batch_size: nil, mapper: nil, release: nil, progress_every: PROGRESS_INTERVAL) ⇒ Runner

Returns a new instance of Runner.



17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/overture_maps/import/runner.rb', line 17

def initialize(model_class:, theme: nil, type: nil, batch_size: nil, mapper: nil,
               release: nil, progress_every: PROGRESS_INTERVAL)
  @model_class = model_class
  @theme = theme
  @type = type
  @release = release
  @batch_size = batch_size || OvertureMaps.configuration.batch_size
  @mapper = mapper
  @progress_every = progress_every
  @imported_count = 0
  @error_count = 0
  @errors = []
end

Instance Attribute Details

#batch_size ⇒ Object (readonly)

Returns the value of attribute batch_size.



15
16
17
 
# File 'lib/overture_maps/import/runner.rb', line 15

def batch_size
  @batch_size
end

#error_count ⇒ Object (readonly)

Returns the value of attribute error_count.



15
16
17
 
# File 'lib/overture_maps/import/runner.rb', line 15

def error_count
  @error_count
end

#errors ⇒ Object (readonly)

Returns the value of attribute errors.



15
16
17
 
# File 'lib/overture_maps/import/runner.rb', line 15

def errors
  @errors
end

#imported_count ⇒ Object (readonly)

Returns the value of attribute imported_count.



15
16
17
 
# File 'lib/overture_maps/import/runner.rb', line 15

def imported_count
  @imported_count
end

#model_class ⇒ Object (readonly)

Returns the value of attribute model_class.



15
16
17
 
# File 'lib/overture_maps/import/runner.rb', line 15

def model_class
  @model_class
end

#theme ⇒ Object (readonly)

Returns the value of attribute theme.



15
16
17
 
# File 'lib/overture_maps/import/runner.rb', line 15

def theme
  @theme
end

#type ⇒ Object (readonly)

Returns the value of attribute type.



15
16
17
 
# File 'lib/overture_maps/import/runner.rb', line 15

def type
  @type
end

Instance Method Details

#import_from_file(path, theme: nil, transform: nil, filter: nil) ⇒ Object 



56
57
58
59
 
# File 'lib/overture_maps/import/runner.rb', line 56

def import_from_file(path, theme: nil, transform: nil, filter: nil)
  reader = ParquetReader.new(theme: theme)
  import_from_reader(reader, source: path, transform: transform, filter: filter)
end

#import_from_reader(reader, source:, transform: nil, filter: nil) ⇒ Object 



52
53
54
 
# File 'lib/overture_maps/import/runner.rb', line 52

def import_from_reader(reader, source:, transform: nil, filter: nil)
  import_from_records(reader.enum_for(:each_record, source: source), transform: transform, filter: filter)
end

#import_from_records(records, transform: nil, filter: nil) ⇒ Object

Imports from anything that yields raw Overture record hashes.



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/overture_maps/import/runner.rb', line 32

def import_from_records(records, transform: nil, filter: nil)
  batch = []

  records.each do |record|
    next if filter && !filter.call(record)

    attrs = transform_record(record, transform)
    next unless attrs

    batch << attrs
    if batch.length >= batch_size
      flush_records(batch)
      batch = []
    end
  end

  flush_records(batch) if batch.any?
  self
end

#success? ⇒ Boolean

Returns:

  • (Boolean) 


61
62
63
 
# File 'lib/overture_maps/import/runner.rb', line 61

def success?
  @error_count.zero?
end