Module: CanvasSync

Defined in:
lib/canvas_sync.rb,
lib/canvas_sync/job.rb,
lib/canvas_sync/config.rb,
lib/canvas_sync/engine.rb,
lib/canvas_sync/record.rb,
lib/canvas_sync/version.rb,
lib/canvas_sync/live_events.rb,
lib/canvas_sync/misc_helper.rb,
lib/canvas_sync/reports_map.rb,
lib/canvas_sync/sidekiq_job.rb,
app/models/canvas_sync/job_log.rb,
lib/canvas_sync/batch_processor.rb,
app/models/canvas_sync/sync_batch.rb,
lib/canvas_sync/jobs/report_starter.rb,
lib/canvas_sync/jobs/sync_roles_job.rb,
lib/canvas_sync/jobs/sync_terms_job.rb,
lib/canvas_sync/jobs/sync_admins_job.rb,
lib/canvas_sync/jobs/sync_scores_job.rb,
lib/canvas_sync/jobs/report_sync_task.rb,
lib/canvas_sync/jobs/sync_rubrics_job.rb,
lib/canvas_sync/jobs/term_batches_job.rb,
lib/canvas_sync/jobs/sync_accounts_job.rb,
lib/canvas_sync/class_callback_executor.rb,
lib/canvas_sync/importers/bulk_importer.rb,
lib/canvas_sync/importers/legacy_importer.rb,
lib/canvas_sync/jobs/begin_sync_chain_job.rb,
lib/canvas_sync/jobs/sync_assignments_job.rb,
lib/canvas_sync/jobs/sync_submissions_job.rb,
lib/canvas_sync/jobs/sync_simple_table_job.rb,
lib/canvas_sync/processors/normal_processor.rb,
lib/canvas_sync/processors/report_processor.rb,
lib/canvas_sync/generators/install_generator.rb,
lib/canvas_sync/jobs/sync_context_modules_job.rb,
lib/canvas_sync/jobs/sync_assignment_groups_job.rb,
lib/canvas_sync/jobs/sync_course_progresses_job.rb,
lib/canvas_sync/jobs/sync_content_migrations_job.rb,
lib/canvas_sync/jobs/sync_rubric_assessments_job.rb,
lib/canvas_sync/jobs/sync_provisioning_report_job.rb,
lib/canvas_sync/jobs/sync_rubric_associations_job.rb,
lib/canvas_sync/jobs/sync_assignment_overrides_job.rb,
lib/canvas_sync/jobs/sync_context_module_items_job.rb,
lib/canvas_sync/generators/install_live_events_generator.rb

Defined Under Namespace

Modules: Api, ApiSyncable, Concerns, Importers, Jobs, LiveEvents, MiscHelper, Processors, Record, ReportsMap, Sidekiq Classes: BatchProcessor, ClassCallbackExecutor, Config, Engine, InstallGenerator, InstallLiveEventsGenerator, Job, JobLog, SyncBatch

Constant Summary collapse

JobBatches = 
Joblin::Batching
JobUniqueness = 
Joblin::Uniqueness
KNOWN_MODELS = 
{
  users: { job: :provisioning, term_scoped: false },
  pseudonyms: { job: :provisioning, term_scoped: false },
  user_observers: { job: :provisioning, term_scoped: false },
  grading_periods: { job: :provisioning, term_scoped: false },
  grading_period_groups: { job: :provisioning, term_scoped: false },
  learning_outcomes: { job: :provisioning, term_scoped: false },
  learning_outcome_results: { job: :provisioning, term_scoped: false },

  terms: { job: CanvasSync::Jobs::SyncTermsJob, term_scoped: false },
  accounts: { job: CanvasSync::Jobs::SyncAccountsJob, term_scoped: false },
  roles: { job: CanvasSync::Jobs::SyncRolesJob, term_scoped: false },
  admins: { job: CanvasSync::Jobs::SyncAdminsJob, term_scoped: false },

  groups: { job: :provisioning, term_scoped: false },
  group_memberships: { job: :provisioning, term_scoped: false },

  courses: { job: :provisioning, term_scoped: true },
  course_nicknames: { job: :provisioning, term_scoped: true },
  sections: { job: :provisioning, term_scoped: true },
  xlist: { job: :provisioning, term_scoped: true },

  context_modules: { job: CanvasSync::Jobs::SyncContextModulesJob, term_scoped: true },
  context_module_items: { job: CanvasSync::Jobs::SyncContextModuleItemsJob, term_scoped: true },
  content_migrations: { job: CanvasSync::Jobs::SyncContentMigrationsJob, term_scoped: true },

  enrollments: { job: :provisioning, term_scoped: true },
  course_progresses: { job: CanvasSync::Jobs::SyncCourseProgressesJob, term_scoped: true },

  assignments: { job: CanvasSync::Jobs::SyncAssignmentsJob, term_scoped: true },
  assignment_groups: { job: CanvasSync::Jobs::SyncAssignmentGroupsJob, term_scoped: true },
  assignment_overrides: { job: CanvasSync::Jobs::SyncAssignmentOverridesJob, term_scoped: true },
  submissions: { job: CanvasSync::Jobs::SyncSubmissionsJob, term_scoped: true },
  scores: { job: CanvasSync::Jobs::SyncScoresJob, term_scoped: true },

  rubrics: { job: CanvasSync::Jobs::SyncRubricsJob, term_scoped: false },
  rubric_associations: { job: CanvasSync::Jobs::SyncRubricAssociationsJob, term_scoped: false },
  rubric_assessments: { job: CanvasSync::Jobs::SyncRubricAssessmentsJob, term_scoped: false },
}.with_indifferent_access.freeze
SUPPORTED_MODELS = 
KNOWN_MODELS.keys.map(&:to_s).freeze
SUPPORTED_TERM_SCOPE_MODELS = 
KNOWN_MODELS.select { |_, v| v[:term_scoped] }.keys.map(&:to_s).freeze
DEFAULT_TERM_SCOPE_MODELS = 
SUPPORTED_TERM_SCOPE_MODELS
PROVISIONING_REPORT_MODELS = 
KNOWN_MODELS.select { |_, v| v[:job] == :provisioning }.keys.map(&:to_s).freeze
SUPPORTED_LIVE_EVENTS = 
%w[
  course
  enrollment
  submission
  assignment
  user
  syllabus
  grade
  module
  module_item
  course_section
].freeze
SUPPORTED_NON_PROV_REPORTS = 
%w[
  graded_submissions
].freeze
VERSION = 
"0.27.15".freeze

Class Method Summary collapse

Class Method Details

.base_canvas_sync_chain(legacy_support: false, account_id: nil, updated_after: nil, full_sync_every: nil, batch_genre: nil, globals: {}, &blk) ⇒ Object 



251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
 
# File 'lib/canvas_sync.rb', line 251

def base_canvas_sync_chain(
  legacy_support: false, # Import records 1 by 1 instead of with bulk upserts
  account_id: nil, # legacy/non PandaPal apps
  updated_after: nil,
  full_sync_every: nil,
  batch_genre: nil,
  globals: {},
  &blk
)
  global_options = {
    legacy_support: legacy_support,
    updated_after: updated_after,
    full_sync_every: full_sync_every,
    batch_genre: batch_genre,
  }
  global_options[:account_id] =  if .present?
  global_options.merge!(globals) if globals

  JobBatches::ChainBuilder.build(CanvasSync::Jobs::BeginSyncChainJob, *[], **global_options, &blk)
end

.config ⇒ Object

Returns the CanvasSync config



339
340
341
 
# File 'lib/canvas_sync.rb', line 339

def config
  @config ||= CanvasSync::Config.new
end

.configure {|config| ... } ⇒ Object

Configure options for CanvasSync. See config.rb for valid configuration options.

Example:

CanvasSync.configure do |config| config.classes_to_only_log_errors_on << "Blah" end

Yields:



333
334
335
336
 
# File 'lib/canvas_sync.rb', line 333

def configure
  yield config
  config
end

.default_provisioning_report_chain(models, term_scope: nil, term_scoped_models: DEFAULT_TERM_SCOPE_MODELS, options: {}, **kwargs) ⇒ Hash

Syncs terms, users/roles/admins if necessary, then the rest of the specified models.

Parameters:

  • models (Array<String>)
  • term_scope (String) (defaults to: nil)
  • legacy_support (Boolean, false) —

    This enables legacy_support, where rows are not bulk inserted. For this to work your models must have a create_or_udpate_from_csv class method that takes a row and inserts it into the database.

  • account_id (Integer, nil) —

    This optional parameter can be used if your Term creation and canvas_sync_client methods require an account ID.

Returns:

  • (Hash)

Raises:

  • (ArgumentError) 


159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
 
# File 'lib/canvas_sync.rb', line 159

def default_provisioning_report_chain(
  models,
  term_scope: nil,
  term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
  options: {},
  **kwargs
) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
  return unless models.present?
  models.map! &:to_s
  term_scope = term_scope.to_s if term_scope
  options = options.deep_symbolize_keys!

  given_models = models.dup
  term_scoped_models = term_scoped_models & SUPPORTED_TERM_SCOPE_MODELS

  root_chain = base_canvas_sync_chain(**kwargs, globals: options[:global] || kwargs[:globals])
  concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
  concurrent_root_chain.kwargs[:description] = "Default Concurrent Batch"
  root_chain << concurrent_root_chain
  current_chain = concurrent_root_chain

  try_add_model_job = ->(model) {
    return unless models.include?(model)
    return if KNOWN_MODELS[model][:job] == :provisioning
    current_chain << { job: KNOWN_MODELS[model][:job].to_s, options: options[model.to_sym] || {} }
    models -= [model]
  }

  ##############################
  # General provisioning jobs (not term-scoped)
  ##############################

  models -= ['terms']

  # Accounts, users, roles, and admins cannot be scoped to term
  try_add_model_job.call('accounts')

  # We could safely remove the unless wrapping, but it allows us to just merge these models into
  #  the later provisioning report so we do it w/ one report instead of two.
  unless term_scope == false
    # These Models use the provisioning report, but are not term-scoped,
    # so we sync them outside of the term scoping to ensure work is not duplicated
    first_provisioning_models = (models - term_scoped_models) & PROVISIONING_REPORT_MODELS
    models -= first_provisioning_models
    current_chain.insert(generate_provisioning_jobs(first_provisioning_models, options))
  end

  try_add_model_job.call('roles')
  try_add_model_job.call('admins')

  # (SUPPORTED_TERM_SCOPE_MODELS - term_scoped_models).each do |mdl|
  (models - term_scoped_models).each do |mdl|
    try_add_model_job.call(mdl)
  end

  ###############################
  # Per-term provisioning jobs
  ###############################

  term_parent_chain = current_chain

  per_term_chain = JobBatches::ChainBuilder.build(KNOWN_MODELS['terms'][:job].to_s, term_scope: term_scope)
  current_chain = per_term_chain

  term_scoped_models.each do |mdl|
    try_add_model_job.call(mdl)
  end

  prov_rep_models = models & PROVISIONING_REPORT_MODELS
  models -= prov_rep_models
  current_chain.insert(
    generate_provisioning_jobs(prov_rep_models, options)
  )

  # Skip syncing terms if not required
  if !current_chain.empty? || (given_models & ['terms']).present?
    term_parent_chain << per_term_chain
  end

  raise ArgumentError, "Some models were not processed: #{models.join(', ')}" if models.present?

  ###############################
  # Wrap it all up
  ###############################

  if given_models.include?('pseudonyms')
    root_chain << Concerns::UserViaPseudonym::RefreshUserCachesJob
  end

  root_chain
end

.generate_provisioning_jobs(model_list, options_hash, job_options: {}, only_split: nil, default_key: :provisioning) ⇒ Object 



293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
 
# File 'lib/canvas_sync.rb', line 293

def generate_provisioning_jobs(model_list, options_hash, job_options: {}, only_split: nil, default_key: :provisioning)
  # Group the model options as best we can.
  # This is mainly for backwards compatibility, since 'users' was previously it's own job
  unique_option_models = group_by_job_options(
    model_list,
    options_hash,
    only_split: only_split,
    default_key: default_key,
  )

  unique_option_models.map do |mopts, models|
    opts = { models: models }
    opts.merge!(job_options)
    opts.merge!(mopts) if mopts.present?
    {
      job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s,
      options: opts,
    }
  end
end

.get_canvas_sync_client(options) ⇒ Object

Calls the canvas_sync_client in your app. If you have specified an account ID when starting the job it will pass the account ID to your canvas_sync_client method.

Parameters:

  • options (Hash) 


318
319
320
321
322
323
324
 
# File 'lib/canvas_sync.rb', line 318

def get_canvas_sync_client(options)
  if options[:account_id]
    canvas_sync_client(options[:account_id])
  else
    canvas_sync_client
  end
end

.group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning) ⇒ Object 



272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
 
# File 'lib/canvas_sync.rb', line 272

def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
  dup_models = [ *model_list ]
  unique_option_models = {}

  filtered_models = only_split ? (only_split & model_list) : model_list
  filtered_models.each do |m|
    mopts = options_hash[m.to_sym] || options_hash[default_key]
    unique_option_models[mopts] ||= []
    unique_option_models[mopts] << m
    dup_models.delete(m)
  end

  if dup_models.present?
    mopts = options_hash[default_key]
    unique_option_models[mopts] ||= []
    unique_option_models[mopts].concat(dup_models)
  end

  unique_option_models
end

.logger ⇒ Object 



355
356
357
358
359
360
 
# File 'lib/canvas_sync.rb', line 355

def logger
  return @logger if defined? @logger
  @logger = Logger.new(STDOUT)
  @logger.level = Logger::WARN
  @logger
end

.provisioning_sync(models, **kwargs) ⇒ Object

Runs a standard provisioning sync job with no extra report types. Terms will be synced first using the API. If you are syncing users/roles/admins and have also specified a Term scope, Users/Roles/Admins will by synced first, before every other model (as Users/Roles/Admins are never scoped to Term).

Parameters:

  • models (Array<String>) —

    A list of models to sync. e.g., ['users', 'courses']. must be one of SUPPORTED_MODELS

  • term_scope (Symbol, nil) —

    An optional symbol representing a scope that exists on the Term model. The provisioning report will be run for each of the terms contained in that scope.

  • legacy_support (Boolean | Array<String>, false) —

    This enables legacy_support, where rows are not bulk inserted. For this to work your models must have a create_or_udpate_from_csv class method that takes a row and inserts it into the database. If an array of model names is provided then only those models will use legacy support.

  • account_id (Integer, nil) —

    This optional parameter can be used if your Term creation and canvas_sync_client methods require an account ID.



114
115
116
117
 
# File 'lib/canvas_sync.rb', line 114

def provisioning_sync(models, **kwargs)
  validate_models!(models)
  default_provisioning_report_chain(models, **kwargs).process!
end

.redis(&blk) ⇒ Object 



362
363
364
 
# File 'lib/canvas_sync.rb', line 362

def redis(&blk)
  ::Bearcat.redis(&blk)
end

.redis_prefix ⇒ Object 



366
367
368
369
370
 
# File 'lib/canvas_sync.rb', line 366

def redis_prefix
  pfx = config.redis_key_prefix
  pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
  pfx
end

.require_report!(*args, **kwargs) ⇒ Object 



119
120
121
 
# File 'lib/canvas_sync.rb', line 119

def require_report!(*args, **kwargs)
  ReportsMap.require_report!(*args, **kwargs)
end

.sync_scope(scope, fallback_scopes: []) ⇒ Object

Given a Model or Relation, scope it down to items that should be synced



124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/canvas_sync.rb', line 124

def sync_scope(scope, fallback_scopes: [])
  terms = [
    :should_canvas_sync,
    :active_for_canvas_sync,
    :should_sync,
    :active_for_sync,
    *fallback_scopes,
    :active,
  ]
  terms.each do |t|
    return scope.send(t) if scope.respond_to?(t)
  end

  if block_given? && !(block_result = yield).nil?
    return block_result
  end

  model = scope.try(:model) || scope
  if model.try(:column_names)&.include?("workflow_state")
    return scope.where.not(workflow_state: %w[deleted])
  end
  Rails.logger.warn("Could not filter Syncable Scope for model '#{scope.try(:model)&.name || scope.name}'")
  scope
end

.validate_live_events!(events) ⇒ Object 



349
350
351
352
353
 
# File 'lib/canvas_sync.rb', line 349

def validate_live_events!(events)
  invalid = events - SUPPORTED_LIVE_EVENTS
  return if invalid.empty?
  raise "Invalid live event(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_LIVE_EVENTS.join(', ')} are supported."
end

.validate_models!(models) ⇒ Object 



343
344
345
346
347
 
# File 'lib/canvas_sync.rb', line 343

def validate_models!(models)
  invalid = models - SUPPORTED_MODELS
  return if invalid.empty?
  raise "Invalid model(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_MODELS.join(', ')} are supported."
end