Module: HasHelpers::DN::Utils::BackfillUtils

Defined in:
app/lib/has_helpers/dn/utils/backfill_utils.rb

Constant Summary collapse

BATCH_SIZE =

default batch size

1000
ID_BATCH_SIZE = 
300
LOCKED = 
"LOCKED"
STAT_TYPE = 
"denormalization"
STAT_SUBTYPE = 
"dn_utils"

Instance Method Summary collapse

Instance Method Details

#define_where_clause(start_updated_at, end_updated_at, organization_id) ⇒ Object 



55
56
57
58
59
60
61
62
63
 
# File 'app/lib/has_helpers/dn/utils/backfill_utils.rb', line 55

def define_where_clause(start_updated_at, end_updated_at, organization_id)
  # set the where clause for backfills withoud ids
  use_where_clause = start_updated_at.present? || end_updated_at.present? || organization_id.present?
  start_where_clause = "updated_at > '#{start_updated_at}'" if start_updated_at.present?
  end_where_clause = "updated_at < '#{end_updated_at}'" if end_updated_at.present?
  org_clause = "organization_id = #{organization_id}" if organization_id.present?
  where_clause = use_where_clause ? "WHERE " + [start_where_clause, end_where_clause, org_clause].compact.join(" AND ") : [start_where_clause, end_where_clause, org_clause].compact.join(" AND ")
  where_clause
end

#get_records_to_update_count(resource, where_clause) ⇒ Object 



65
66
67
 
# File 'app/lib/has_helpers/dn/utils/backfill_utils.rb', line 65

def get_records_to_update_count(resource, where_clause)
  where_clause.blank? ? resource.count : resource.where(where_clause.gsub("WHERE", "")).count
end

#prepare_sql_statement(db_columns, resource, **options) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'app/lib/has_helpers/dn/utils/backfill_utils.rb', line 35

def prepare_sql_statement(db_columns, resource, **options)
  # errors = []
  options = options.symbolize_keys
  columns = options[:columns] || nil
  start_updated_at = options[:start_updated_at] || nil
  end_updated_at = options[:end_updated_at] || nil
  ids_to_update = options[:ids_to_update] || nil
  organization_id = options[:organization_id] || nil
  # set the columns to update
  update_statement = columns.present? ? columns.map { |c| db_columns[c.to_sym] }.join(",") : db_columns.values.join(",")
  where_clause = define_where_clause(start_updated_at, end_updated_at, organization_id) if ids_to_update.nil?
  records_to_update = ids_to_update.present? ? ids_to_update.size : get_records_to_update_count(resource, where_clause)
  save_to_stats(resource.table_name, STAT_SUBTYPE, { table: resource.table_name, count: records_to_update })

  # Enqueue in batches
  Array(options[:ids_to_update]).each_slice(BATCH_SIZE) do |batch_ids|
    ::GoodJob::DNUpdateWorker.perform_deferred(resource.table_name, batch_ids, update_statement, options[:disable_triggers])
  end
end

#save_to_stats(table_name, subtype, data) ⇒ Object 



69
70
71
72
73
74
75
76
 
# File 'app/lib/has_helpers/dn/utils/backfill_utils.rb', line 69

def save_to_stats(table_name, subtype, data)
  stat = ::HasHelpers::Stat.new
  stat.name = table_name
  stat.maintype = STAT_TYPE
  stat.subtype = subtype
  stat.data = data
  stat.save!
end

#update_sql(table_name, offset, update_statement, where_clause, ids_to_update = nil, disable_triggers = true) ⇒ Object 



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'app/lib/has_helpers/dn/utils/backfill_utils.rb', line 14

def update_sql(table_name, offset, update_statement, where_clause, ids_to_update = nil, disable_triggers = true)
  disable_trigger_statement = disable_triggers ? "SELECT set_config('psql_hq.skip_triggers.ALL', 'true', true);" : nil
  inner_id_clause =
    if ids_to_update.present?
      "'#{ids_to_update[offset, ID_BATCH_SIZE].join("','")}'"
    else
      "SELECT id FROM #{table_name}
      #{where_clause}
      ORDER BY created_at DESC, id
      LIMIT #{BATCH_SIZE} OFFSET #{offset}"
    end
  <<~SQL
    #{disable_trigger_statement}
    UPDATE #{table_name}
    SET #{update_statement}
    WHERE #{table_name}.id IN (
      #{inner_id_clause}
    );
  SQL
end