Class: Noiseless::BulkImporter

Inherits:
Object
  • Object
show all
Defined in:
lib/noiseless/bulk_importer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(model_class, connection: nil) ⇒ BulkImporter

Returns a new instance of BulkImporter.



7
8
9
10
11
 
# File 'lib/noiseless/bulk_importer.rb', line 7

def initialize(model_class, connection: nil)
  @model_class = model_class
  @connection = connection || model_class.connection
  @errors = []
end

Instance Attribute Details

#errors ⇒ Object (readonly)

Returns the value of attribute errors.



5
6
7
 
# File 'lib/noiseless/bulk_importer.rb', line 5

def errors
  @errors
end

#model_class ⇒ Object (readonly)

Returns the value of attribute model_class.



5
6
7
 
# File 'lib/noiseless/bulk_importer.rb', line 5

def model_class
  @model_class
end

Instance Method Details

#import(relation_or_records = nil, batch_size: 1000, transform: nil, preprocess: nil, force: false, refresh: true) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/noiseless/bulk_importer.rb', line 13

def import(relation_or_records = nil,
           batch_size: 1000,
           transform: nil,
           preprocess: nil,
           force: false,
           refresh: true,
           **)
  @errors.clear

  # Create index if force is true
  if force
    delete_index
    create_index
  end

  # Get records to import
  records = resolve_records(relation_or_records)

  total_imported = 0

  records.each_slice(batch_size) do |batch|
    # Apply preprocessing to the entire batch
    processed_batch = preprocess ? preprocess.call(batch) : batch

    # Transform individual records and build actions
    actions = build_bulk_actions(processed_batch, transform)

    # Execute bulk operation
    begin
      client = Noiseless.connections.client(@connection)
      response = client.bulk(actions, refresh: refresh, **)

      # Check for errors in response
      collect_errors(response, processed_batch)

      total_imported += actions.size
    rescue StandardError => e
      @errors << {
        error: e.message,
        batch: processed_batch.map { |r| identify_record(r) }
      }
    end
  end

  {
    imported: total_imported,
    errors: @errors.size,
    error_details: @errors
  }
end

#import_scoped(scope) ⇒ Object 



64
65
66
 
# File 'lib/noiseless/bulk_importer.rb', line 64

def import_scoped(scope, **)
  import(scope, **)
end

#reindex(batch_size: 1000) ⇒ Object

Raises:

  • (ArgumentError) 


68
69
70
71
72
 
# File 'lib/noiseless/bulk_importer.rb', line 68

def reindex(batch_size: 1000, **)
  raise ArgumentError, "Model class #{model_class} must respond to :all for reindexing" unless model_class.respond_to?(:all)

  import(model_class.all, batch_size: batch_size, force: true, **)
end