Class: Tapsoob::Push

Inherits:
Operation show all
Defined in:
lib/tapsoob/operation.rb

Instance Attribute Summary

Attributes inherited from Operation

#database_url, #dump_path, #opts

Instance Method Summary collapse

Methods inherited from Operation

#apply_table_filter, #catch_errors, #completed_tables, #data?, #db, #default_chunksize, #exclude_tables, #exiting?, factory, #format_number, #indexes_first?, #initialize, #log, #resuming?, #schema?, #setup_signal_trap, #store_session, #stream_state, #stream_state=, #table_filter

Constructor Details

This class inherits a constructor from Tapsoob::Operation

Instance Method Details

#fetch_local_tables_info ⇒ Object 



534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
 
# File 'lib/tapsoob/operation.rb', line 534

def fetch_local_tables_info
  tables_with_counts = {}
  tbls = Dir.glob(File.join(dump_path, "schemas", "*")).map { |path| File.basename(path, ".rb") }
  tbls.each do |table|
    if File.exist?(File.join(dump_path, "data", "#{table}.json"))
      # Read NDJSON format - each line is a separate JSON chunk
      total_rows = 0
      File.readlines(File.join(dump_path, "data", "#{table}.json")).each do |line|
        chunk = JSON.parse(line.strip)
        total_rows += chunk["data"].size if chunk["data"]
      end
      tables_with_counts[table] = total_rows
    else
      tables_with_counts[table] = 0
    end
  end
  apply_table_filter(tables_with_counts)
end

#file_prefix ⇒ Object 



357
358
359
 
# File 'lib/tapsoob/operation.rb', line 357

def file_prefix
  "push"
end

#local_tables_info ⇒ Object 



517
518
519
 
# File 'lib/tapsoob/operation.rb', line 517

def local_tables_info
  opts[:local_tables_info] ||= fetch_local_tables_info
end

#push_data ⇒ Object 



435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
 
# File 'lib/tapsoob/operation.rb', line 435

def push_data
  log.info "Sending data"

  log.info "#{tables.size} tables, #{format_number(record_count)} records"

  tables.each do |table_name, count|
    # Skip if data file doesn't exist or has no data
    data_file = File.join(dump_path, "data", "#{table_name}.json")
    next unless File.exist?(data_file) && count > 0
    db[table_name.to_sym].truncate if @opts[:purge]
    stream = Tapsoob::DataStream.factory(db, {
      :table_name => table_name,
      :chunksize => default_chunksize
    }, {
      :"skip-duplicates" => opts[:"skip-duplicates"] || false,
      :"discard-identity" => opts[:"discard-identity"] || false,
      :purge => opts[:purge] || false,
      :debug => opts[:debug]
    })
    estimated_chunks = [(count.to_f / default_chunksize).ceil, 1].max
    progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, estimated_chunks) : nil)
    push_data_from_file(stream, progress)
  end
end

#push_data_from_file(stream, progress) ⇒ Object 



460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
 
# File 'lib/tapsoob/operation.rb', line 460

def push_data_from_file(stream, progress)
  loop do
    if exiting?
      store_session
      exit 0
    end

    row_size = 0
    chunksize = stream.state[:chunksize]

    begin
      chunksize = Tapsoob::Utils.calculate_chunksize(chunksize) do |c|
        stream.state[:chunksize] = c.to_i
        encoded_data, row_size, elapsed_time = nil
        d1 = c.time_delta do
          encoded_data, row_size, elapsed_time = stream.fetch({ :type => "file", :source => dump_path })
        end

        data = nil
        d2 = c.time_delta do
          data = {
            :state        => stream.to_hash,
            :checksum     => Tapsoob::Utils.checksum(encoded_data).to_s,
            :encoded_data => encoded_data
          }
        end

        stream.fetch_data_to_database(data)
        log.debug "row size: #{row_size}"
        self.stream_state = stream.to_hash

        c.idle_secs = (d1 + d2)

        elapsed_time
      end
    rescue Tapsoob::CorruptedData => e
      # retry the same data, it got corrupted somehow.
      next
    rescue Tapsoob::DuplicatePrimaryKeyError => e
      # verify the stream and retry it
      stream.verify_stream
      stream = JSON.generate({ :state => stream.to_hash })
      next
    end
    stream.state[:chunksize] = chunksize

    # Update progress bar by 1 chunk
    progress.inc(1) if progress

    break if stream.complete?
  end

  progress.finish if progress
  completed_tables << stream.table_name.to_s
  self.stream_state = {}
end

#push_indexes ⇒ Object 



379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
 
# File 'lib/tapsoob/operation.rb', line 379

def push_indexes
  idxs = {}
  table_idxs = Dir.glob(File.join(dump_path, "indexes", "*.json")).map { |path| File.basename(path, '.json') }
  table_idxs.each do |table_idx|
    # Read NDJSON format - each line is a separate index
    index_file = File.join(dump_path, "indexes", "#{table_idx}.json")
    idxs[table_idx] = File.readlines(index_file).map { |line| JSON.parse(line.strip) }
  end

  return unless idxs.size > 0

  log.info "Sending indexes"

  apply_table_filter(idxs).each do |table, indexes|
    next unless indexes.size > 0
    progress = ProgressBar.new(table, indexes.size)
    indexes.each do |idx|
      Tapsoob::Utils.load_indexes(database_url, idx)
      progress.inc(1)
    end
    progress.finish
  end
end

#push_partial_data ⇒ Object 



422
423
424
425
426
427
428
429
430
431
432
433
 
# File 'lib/tapsoob/operation.rb', line 422

def push_partial_data
  return if stream_state == {}

  table_name = stream_state[:table_name]
  record_count = tables[table_name.to_s]
  log.info "Resuming #{table_name}, #{format_number(record_count)} records"
  stream = Tapsoob::DataStream.factory(db, stream_state)
  chunksize = stream_state[:chunksize] || default_chunksize
  estimated_chunks = [(record_count.to_f / chunksize).ceil, 1].max
  progress = (opts[:progress] ? ProgressBar.new(table_name.to_s, estimated_chunks) : nil)
  push_data_from_file(stream, progress)
end

#push_reset_sequences ⇒ Object 



416
417
418
419
420
 
# File 'lib/tapsoob/operation.rb', line 416

def push_reset_sequences
  log.info "Resetting sequences"

  Tapsoob::Utils.schema_bin(:reset_db_sequences, database_url)
end

#push_schema ⇒ Object 



403
404
405
406
407
408
409
410
411
412
413
414
 
# File 'lib/tapsoob/operation.rb', line 403

def push_schema
  log.info "Sending schema"

  progress = ProgressBar.new('Schema', tables.size)
  tables.each do |table, count|
    log.debug "Loading '#{table}' schema\n"
    # Reuse existing db connection for better performance
    Tapsoob::Utils.load_schema(dump_path, db, table)
    progress.inc(1)
  end
  progress.finish
end

#record_count ⇒ Object 



530
531
532
 
# File 'lib/tapsoob/operation.rb', line 530

def record_count
  @record_count ||= local_tables_info.values.inject(0) { |a,c| a += c }
end

#run ⇒ Object 



365
366
367
368
369
370
371
372
373
374
375
376
377
 
# File 'lib/tapsoob/operation.rb', line 365

def run
  catch_errors do
    unless resuming?
      push_schema if schema?
      push_indexes if indexes_first? && schema?
    end
    setup_signal_trap
    push_partial_data if data? && resuming?
    push_data if data?
    push_indexes if !indexes_first? && schema?
    push_reset_sequences
  end
end

#tables ⇒ Object 



521
522
523
524
525
526
527
528
 
# File 'lib/tapsoob/operation.rb', line 521

def tables
  h = {}
  local_tables_info.each do |table_name, count|
    next if completed_tables.include?(table_name.to_s)
    h[table_name.to_s] = count
  end
  h
end

#to_hash ⇒ Object 



361
362
363
 
# File 'lib/tapsoob/operation.rb', line 361

def to_hash
  super.merge(:local_tables_info => local_tables_info)
end