Class: HasHelpers::DataMigration::Base

Inherits:
Object
  • Object
show all
Extended by:
S3Utils
Defined in:
lib/has_helpers/data_migration/base.rb

Defined Under Namespace

Classes: Error

Constant Summary collapse

LARGE_MIGRATION_THRESHOLD = 
12.hours

Class Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from S3Utils

get_s3_bucket, get_s3_file

Class Attribute Details

.depends_on=(value) ⇒ Object (writeonly)

Sets the attribute depends_on

Parameters:

  • value —

    the value to set the attribute depends_on to.



98
99
100
 
# File 'lib/has_helpers/data_migration/base.rb', line 98

def depends_on=(value)
  @depends_on = value
end

.environments ⇒ Object 



260
261
262
263
264
265
266
 
# File 'lib/has_helpers/data_migration/base.rb', line 260

def environments
  @environments.present? ? Array(@environments).map(&:to_s) : [::HasUtils::Env::DEVELOPMENT,
                                                               ::HasUtils::Env::PREVIEW,
                                                               ::HasUtils::Env::PRODUCTION,
                                                               ::HasUtils::Env::STAGING,
                                                               ::HasUtils::Env::TEST]
end

.expected_duration ⇒ Object 



244
245
246
 
# File 'lib/has_helpers/data_migration/base.rb', line 244

def expected_duration
  @expected_duration || 24.hours
end

.idempotent=(value) ⇒ Object (writeonly)

Sets the attribute idempotent

Parameters:

  • value —

    the value to set the attribute idempotent to.



98
99
100
 
# File 'lib/has_helpers/data_migration/base.rb', line 98

def idempotent=(value)
  @idempotent = value
end

.urgent=(value) ⇒ Object (writeonly)

Sets the attribute urgent

Parameters:

  • value —

    the value to set the attribute urgent to.



98
99
100
 
# File 'lib/has_helpers/data_migration/base.rb', line 98

def urgent=(value)
  @urgent = value
end

Class Method Details

.bucket_name ⇒ Object 



172
173
174
 
# File 'lib/has_helpers/data_migration/base.rb', line 172

def bucket_name
  self.const_get(:BUCKET_NAME)
end

.complete? ⇒ Boolean

Returns:

  • (Boolean) 


204
205
206
 
# File 'lib/has_helpers/data_migration/base.rb', line 204

def complete?
  model.completed_at.present?
end

.continue_in_one_minute ⇒ Object 



105
106
107
108
 
# File 'lib/has_helpers/data_migration/base.rb', line 105

def continue_in_one_minute
  model.update(enqueued_at: ::Time.current)
  worker_klass.set(wait: 1.minute).perform_later(self.name)
end

.custom_format_datetime(datetime) ⇒ Object 



138
139
140
141
142
 
# File 'lib/has_helpers/data_migration/base.rb', line 138

def custom_format_datetime(datetime)
  datetime = datetime.to_s unless datetime.is_a?(String)
  parsed_datetime = Chronic.parse(datetime)
  parsed_datetime.in_time_zone("UTC").strftime("%Y%m%d%H%M") if parsed_datetime
end

.dependencies ⇒ Object 



240
241
242
 
# File 'lib/has_helpers/data_migration/base.rb', line 240

def dependencies
  Array(@depends_on)
end

.dependencies_met? ⇒ Boolean

Returns:

  • (Boolean) 


189
190
191
192
193
194
 
# File 'lib/has_helpers/data_migration/base.rb', line 189

def dependencies_met?
  dependencies.each do |dependency|
    return false unless dependency_has_completed(dependency)
  end
  true
end

.enqueue ⇒ Object 



100
101
102
103
 
# File 'lib/has_helpers/data_migration/base.rb', line 100

def enqueue
  model.update(enqueued_at: ::Time.current)
  worker_klass.perform_later(self.name)
end

.enqueued? ⇒ Boolean

Returns:

  • (Boolean) 


196
197
198
 
# File 'lib/has_helpers/data_migration/base.rb', line 196

def enqueued?
  model.enqueued_at.present?
end

.get_s3_key(migration_model, s3_file_path, file_output = nil) ⇒ Object

Returns the S3 key for the new migration file (rerun) or the output of the migration. s3_file_path_prefix is the prefix of the original S3 file path up to the second level directory. file_name is the name of the original file without the extension. file_name_modifier is the modifier string calculated from the updated_at timestamp. file_type is the file extension (csv or txt). example: shared/4/20220819_relationship_ids/20220819_relationship_ids_0424202.csv example: shared/4/20220819_relationship_ids/20220819_relationship_ids_042420231_output.txt



151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/has_helpers/data_migration/base.rb', line 151

def get_s3_key(migration_model, s3_file_path, file_output = nil)
  modifier = custom_format_datetime(migration_model.updated_at)
  file_s3 = s3_file_path.split("/").last
  file_name = file_s3.split(".")[0...-1].join(".")
  file_type = file_output.nil? ? file_s3.split(".").last : file_output

  s3_file_path_prefix = (s3_file_path.split("/").take 2).join("/")
  s3_key = "#{s3_file_path_prefix}/#{file_name}/#{file_name}_#{modifier}"
  s3_key += "_output" if file_output
  s3_key += ".#{file_type}"
  s3_key
end

.idempotent? ⇒ Boolean

Returns:

  • (Boolean) 


230
231
232
 
# File 'lib/has_helpers/data_migration/base.rb', line 230

def idempotent?
  !!@idempotent
end

.is_large? ⇒ Boolean

Returns:

  • (Boolean) 


248
249
250
 
# File 'lib/has_helpers/data_migration/base.rb', line 248

def is_large?
  expected_duration > LARGE_MIGRATION_THRESHOLD
end

.new_s3_file_path ⇒ Object 



164
165
166
 
# File 'lib/has_helpers/data_migration/base.rb', line 164

def new_s3_file_path
  get_s3_key(model, s3_file_path)
end

.new_s3_output_path ⇒ Object 



168
169
170
 
# File 'lib/has_helpers/data_migration/base.rb', line 168

def new_s3_output_path
  get_s3_key(model, s3_file_path, "txt")
end

.ready? ⇒ Boolean

Returns:

  • (Boolean) 


208
209
210
211
212
213
214
215
216
 
# File 'lib/has_helpers/data_migration/base.rb', line 208

def ready?
  if dependencies_met? && !complete? && run_in_current_environment? && valid_to_start_now?
    if idempotent?
      true
    else
      true unless started?
    end
  end
end

.rename_s3_file ⇒ Object 



176
177
178
179
 
# File 'lib/has_helpers/data_migration/base.rb', line 176

def rename_s3_file
  s3_client = ::HasHelpers::Aws.new_s3_client(credentials_conf: ::HasHelpers::Aws.credentials_conf)
  upload_object(s3_client, bucket_name, new_s3_file_path)
end

.rerun ⇒ Object

Clears out the timestamps for this data migration so it will run again.



111
112
113
114
115
116
117
 
# File 'lib/has_helpers/data_migration/base.rb', line 111

def rerun
  model.update(
    enqueued_at: nil,
    started_at: nil,
    completed_at: nil
  )
end

.rerunable ⇒ Object 



123
124
125
126
127
128
129
130
131
132
 
# File 'lib/has_helpers/data_migration/base.rb', line 123

def rerunable
  raise "This migration is not rerunable." unless self.const_defined?(:BUCKET_NAME) && self.const_defined?(:S3_FILE_PATH)
  raise "Already enqueued" unless enqueued? && started? && complete?
  raise "File not found" if s3_file.blank?
  self.class_eval do
    self.urgent = true
  end
  enqueue
  rename_s3_file
end

.run_in_current_environment? ⇒ Boolean

Returns:

  • (Boolean) 


256
257
258
 
# File 'lib/has_helpers/data_migration/base.rb', line 256

def run_in_current_environment?
  environments.include?(::HasUtils::Env.current_environment)
end

.s3_file ⇒ Object 



119
120
121
 
# File 'lib/has_helpers/data_migration/base.rb', line 119

def s3_file
  @s3_file ||= get_s3_file(bucket_name, s3_file_path)
end

.s3_file_path ⇒ Object 



134
135
136
 
# File 'lib/has_helpers/data_migration/base.rb', line 134

def s3_file_path
  self.const_get(:S3_FILE_PATH)
end

.started? ⇒ Boolean

Returns:

  • (Boolean) 


200
201
202
 
# File 'lib/has_helpers/data_migration/base.rb', line 200

def started?
  model.started_at.present?
end

.upload_object(s3_client, bucket_name, object_key) ⇒ Object 



181
182
183
184
185
186
187
 
# File 'lib/has_helpers/data_migration/base.rb', line 181

def upload_object(s3_client, bucket_name, object_key)
  s3_client.put_object(
    bucket: bucket_name,
    key: object_key,
    body: s3_file
  )
end

.urgent? ⇒ Boolean

In non-production environments, we always return true so data migrations run immediately.

Returns:

  • (Boolean) 


236
237
238
 
# File 'lib/has_helpers/data_migration/base.rb', line 236

def urgent?
  !::HasUtils::Env.production? || !!@urgent
end

.valid_to_start_now? ⇒ Boolean

Returns true if the data migration is urgent, or if we are outside business hours (using US Central Time). Business hours are defined as 7am-7pm M-F (except holidays as defined in the HasHelpers date extensions).

Returns:

  • (Boolean) 


222
223
224
225
226
227
228
 
# File 'lib/has_helpers/data_migration/base.rb', line 222

def valid_to_start_now?
  Time.use_zone("Central Time (US & Canada)") do
    urgent? ||
      !Date.current.business_day? ||
      ::Time.current.seconds_since_midnight < 7.hours || ::Time.current.seconds_until_end_of_day < 5.hours
  end
end

.worker_klass ⇒ Object 



252
253
254
 
# File 'lib/has_helpers/data_migration/base.rb', line 252

def worker_klass
  is_large? ? HasHelpers::DataMigration::LargeWorker : HasHelpers::DataMigration::SmallWorker
end

Instance Method Details

#run ⇒ Object

Raises:

  • (NotImplementedError) 


52
53
54
 
# File 'lib/has_helpers/data_migration/base.rb', line 52

def run
  raise NotImplementedError
end