Class: Chef::Knife::EcImport

Inherits:
Chef::Knife show all
Includes:
EcBase
Defined in:
lib/chef/knife/ec_import.rb

Constant Summary collapse

PUBLIC_KEY_READ_ACCESS_JSON =

Constants for duplicated strings

'public_key_read_access.json'
FROZEN_STATUS_KEY = 
'frozen?'
ADMIN_GROUPS = 
['admins', 'billing-admins'].freeze
ADMIN_GROUP_FILES = 
['billing-admins.json', 'public_key_read_access.json'].freeze
NOT_FOUND_STATUS = 
'404'
PATHS = 
%w(chef_repo_path cookbook_path environment_path data_bag_path role_path node_path client_path acl_path group_path container_path)
PERMISSIONS = 
%w{create read update delete grant}.freeze

Instance Method Summary collapse

Methods included from EcBase

#completion_banner, #configure_chef, #ensure_webui_key_exists!, included, #knife_ec_error_handler, #local_user_list, #org_admin, #remote_user_list, #remote_users, #rest, #server, #set_dest_dir_from_args!, #temporary_webui_key, #users_for_purge, #veil, #veil_config, #warn_on_incorrect_clients_group, #webui_key

Instance Method Details

#chef_fs_copy_pattern(pattern_str, chef_fs_config) ⇒ Object 



281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
 
# File 'lib/chef/knife/ec_import.rb', line 281

def chef_fs_copy_pattern(pattern_str, chef_fs_config)
  ui.msg "Copying #{pattern_str}"
  pattern = Chef::ChefFS::FilePattern.new(pattern_str)
  Chef::ChefFS::FileSystem.copy_to(pattern, chef_fs_config.local_fs,
                                   chef_fs_config.chef_fs, nil,
                                   config, ui,
                                   proc { |entry| chef_fs_config.format_path(entry) })
rescue Net::HTTPClientException,
       Chef::ChefFS::FileSystem::NotFoundError,
       Chef::ChefFS::FileSystem::OperationFailedError,
       Chef::Exceptions::JSON::ParseError,
       JSON::ParserError => ex
  ui.error "#{pattern_str} failed to copy: #{ex.message}"
  knife_ec_error_handler.add(ex)
end

#cookbook_url(org_name, cookbook_name, version, params = nil) ⇒ Object

Helper method to construct cookbook URL



58
59
60
61
 
# File 'lib/chef/knife/ec_import.rb', line 58

def cookbook_url(org_name, cookbook_name, version, params = nil)
  url = org_url(org_name, 'cookbooks', cookbook_name, version)
  params ? "#{url}?#{params}" : url
end

#for_each_organization ⇒ Object 



187
188
189
190
191
192
193
 
# File 'lib/chef/knife/ec_import.rb', line 187

def for_each_organization
  Dir.foreach("#{dest_dir}/organizations") do |name|
    next if name == '..' || name == '.' || !File.directory?("#{dest_dir}/organizations/#{name}")
    next unless (config[:org].nil? || config[:org] == name)
    yield name
  end
end

#for_each_user ⇒ Object 



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/chef/knife/ec_import.rb', line 171

def for_each_user
  Dir.foreach("#{dest_dir}/users") do |filename|
    next if filename !~ /(.+)\.json/
    name = $1
    # We don't have overwrite_pivotal option, but we should probably still skip pivotal if it's in the backup
    # to avoid messing with the system user, although we are only doing ACLs here.
    if name == 'pivotal'
       # In restore, we skip pivotal unless overwrite_pivotal is true.
       # Here we don't have that flag, so we should probably always skip pivotal for safety
       # as we are not managing users.
       next
    end
    yield name
  end
end

#freeze_cookbook(cookbook_name, version, org_name) ⇒ Object 



389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
 
# File 'lib/chef/knife/ec_import.rb', line 389

def freeze_cookbook(cookbook_name, version, org_name)
  ui.msg "Freezing cookbook #{cookbook_name} version #{version}"

  # Get the current cookbook manifest
  manifest = rest.get(cookbook_url(org_name, cookbook_name, version))

  if manifest[FROZEN_STATUS_KEY] # Ignore if already frozen
    ui.warn "Freezing cookbook #{cookbook_name} version #{version} skipped since it is already frozen!"
    return
  end

  rest.put(cookbook_url(org_name, cookbook_name, version, 'freeze=true'), 
           manifest.tap { |h| h[FROZEN_STATUS_KEY] = true })
rescue Net::HTTPClientException => ex
  ui.warn "Failed to freeze cookbook #{cookbook_name} #{version}: #{ex.message}"
  knife_ec_error_handler.add(ex)
end

#get_admin_group_acl_paths(chef_fs_config) ⇒ Object

Helper method to get admin group ACL paths



82
83
84
85
86
 
# File 'lib/chef/knife/ec_import.rb', line 82

def get_admin_group_acl_paths(chef_fs_config)
  acl_paths = ['/acls/groups/billing-admins.json']
  acl_paths.push('/acls/groups/public_key_read_access.json') if public_key_read_access_exists?(chef_fs_config, 'acls')
  acl_paths
end

#get_admin_groups(chef_fs_config) ⇒ Object

Helper method to get admin groups based on what exists



75
76
77
78
79
 
# File 'lib/chef/knife/ec_import.rb', line 75

def get_admin_groups(chef_fs_config)
  groups = ADMIN_GROUPS.dup
  groups.push('public_key_read_access') if public_key_read_access_exists?(chef_fs_config, 'groups')
  groups
end

#group_array_to_sortable_hash(groups) ⇒ Object 



301
302
303
304
305
306
307
308
309
310
311
312
 
# File 'lib/chef/knife/ec_import.rb', line 301

def group_array_to_sortable_hash(groups)
  ret = {}
  groups.each do |group|
    name = group['name']
    ret[name] = if group.key?('groups')
                  group['groups']
                else
                  []
                end
  end
  ret
end

#list_chef_fs_entries(chef_fs_config, pattern, exclude_files = []) ⇒ Object

Helper method to list ChefFS entries with pattern and filter



69
70
71
72
 
# File 'lib/chef/knife/ec_import.rb', line 69

def list_chef_fs_entries(chef_fs_config, pattern, exclude_files = [])
  Chef::ChefFS::FileSystem.list(chef_fs_config.local_fs, Chef::ChefFS::FilePattern.new(pattern))
    .select { |entry| !exclude_files.include?(entry.name) }
end

#org_file_path(orgname, *path_parts) ⇒ Object

Helper method to construct organization file path



47
48
49
 
# File 'lib/chef/knife/ec_import.rb', line 47

def org_file_path(orgname, *path_parts)
  File.join(dest_dir, 'organizations', orgname, *path_parts)
end

#org_url(orgname, *path_parts) ⇒ Object

Helper method to construct organization URL



52
53
54
55
 
# File 'lib/chef/knife/ec_import.rb', line 52

def org_url(orgname, *path_parts)
  path = path_parts.join('/')
  path.empty? ? "organizations/#{orgname}" : "organizations/#{orgname}/#{path}"
end

#organization_exists?(orgname) ⇒ Boolean

Returns:

  • (Boolean) 


146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/chef/knife/ec_import.rb', line 146

def organization_exists?(orgname)
  rest.get(org_url(orgname))
  true
rescue Net::HTTPClientException => ex
  if ex.response.code == NOT_FOUND_STATUS
    false
  else
    knife_ec_error_handler.add(ex)
    false
  end
end

#process_cookbook_frozen_status(cookbooks_path, cookbook_entry, org_name) ⇒ Object 



363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
 
# File 'lib/chef/knife/ec_import.rb', line 363

def process_cookbook_frozen_status(cookbooks_path, cookbook_entry, org_name)
  cookbook_path = File.join(cookbooks_path, cookbook_entry)
  return unless File.directory?(cookbook_path)

  # cookbook_entry is in format "cookbook_name-version"
  # Extract cookbook name and version
  # Use character class negation to prevent backtracking (ReDoS)
  # Match: name-X.Y.Z or name-X.Y.Z.suffix (e.g., mycb-1.0.0 or mycb-1.0.0.beta1)
  return unless cookbook_entry =~ /^([^-]+(?:-[^-]+)*?)-(\d+\.\d+\.\d+(?:\.[^.]+)*)$/
  
  cookbook_name = $1
  version = $2

  status_file = File.join(cookbook_path, 'status.json')
  return unless File.exist?(status_file)

  begin
    status_data = JSON.parse(File.read(status_file))
    freeze_cookbook(cookbook_name, version, org_name) if status_data['frozen'] == true
  rescue JSON::ParserError => e
    ui.warn "Failed to parse status.json for #{cookbook_name} #{version}: #{e.message}"
  rescue => e
    ui.warn "Failed to restore frozen status for #{cookbook_name} #{version}: #{e.message}"
  end
end

#public_key_read_access_exists?(chef_fs_config, type = 'groups') ⇒ Boolean

Helper method to check if public key read access group exists

Returns:

  • (Boolean) 


64
65
66
 
# File 'lib/chef/knife/ec_import.rb', line 64

def public_key_read_access_exists?(chef_fs_config, type = 'groups')
  ::File.exist?(::File.join(chef_fs_config.local_fs.child_paths[type], 'groups', PUBLIC_KEY_READ_ACCESS_JSON))
end

#put_acl(rest, url, acls) ⇒ Object 



408
409
410
411
412
413
414
415
416
417
418
419
 
# File 'lib/chef/knife/ec_import.rb', line 408

def put_acl(rest, url, acls)
  old_acls = rest.get(url)
  old_acls = Chef::ChefFS::DataHandler::AclDataHandler.new.normalize(old_acls, nil)
  acls = Chef::ChefFS::DataHandler::AclDataHandler.new.normalize(acls, nil)
  if acls != old_acls
    PERMISSIONS.each do |permission|
      rest.put("#{url}/#{permission}", { permission => acls[permission] })
    end
  end
rescue Net::HTTPClientException => ex
  knife_ec_error_handler.add(ex)
end

#read_json_file(path) ⇒ Object

Helper method to read JSON file from backup directory



42
43
44
 
# File 'lib/chef/knife/ec_import.rb', line 42

def read_json_file(path)
  JSONCompat.from_json(File.read(path))
end

#restore_cookbook_frozen_status(org_name, chef_fs_config) ⇒ Object 



348
349
350
351
352
353
354
355
356
357
358
359
360
361
 
# File 'lib/chef/knife/ec_import.rb', line 348

def restore_cookbook_frozen_status(org_name, chef_fs_config)
  return if config[:skip_frozen_cookbook_status]

  ui.msg 'Restoring cookbook frozen status'
  cookbooks_path = org_file_path(org_name, 'cookbooks')

  return unless File.directory?(cookbooks_path)

  Dir.foreach(cookbooks_path) do |cookbook_entry|
    next if cookbook_entry == '.' || cookbook_entry == '..'
    
    process_cookbook_frozen_status(cookbooks_path, cookbook_entry, org_name)
  end
end

#restore_group(chef_fs_config, group_name, includes = {:users => true, :clients => true}) ⇒ Object 



314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
 
# File 'lib/chef/knife/ec_import.rb', line 314

def restore_group(chef_fs_config, group_name, includes = {:users => true, :clients => true})
  includes[:users] = true unless includes.key? :users
  includes[:clients] = true unless includes.key? :clients

  ui.msg "Copying /groups/#{group_name}.json"
  group = Chef::ChefFS::FileSystem.resolve_path(
    chef_fs_config.chef_fs,
    "/groups/#{group_name}.json"
  )

  # Will throw NotFoundError if JSON file does not exist on disk. See below.
  members_json = Chef::ChefFS::FileSystem.resolve_path(
    chef_fs_config.local_fs,
    "/groups/#{group_name}.json"
  ).read

  members = JSON.parse(members_json).select do |member|
    if includes[:users] && includes[:clients]
      member
    elsif includes[:users]
      member == 'users'
    elsif includes[:clients]
      member == 'clients'
    end
  end

  group.write(members.to_json)
rescue Chef::ChefFS::FileSystem::NotFoundError
  Chef::Log.warn "Could not find #{group.display_path} on disk. Will not restore."
rescue JSON::ParserError, Chef::Exceptions::JSON::ParseError => ex
  ui.warn "Failed to parse group file #{group_name}.json: #{ex.message}. Skipping group restore."
  knife_ec_error_handler.add(ex)
end

#restore_user_acls ⇒ Object 



158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/chef/knife/ec_import.rb', line 158

def restore_user_acls
  ui.msg 'Restoring user ACLs'
  for_each_user do |name|
    begin
      user_acl = read_json_file(File.join(dest_dir, 'user_acls', "#{name}.json"))
      put_acl(user_acl_rest, "users/#{name}/_acl", user_acl)
    rescue Chef::Exceptions::JSON::ParseError, JSON::ParserError => ex
      ui.warn "Failed to parse user ACL for #{name}: #{ex.message}. Skipping."
      knife_ec_error_handler.add(ex)
    end
  end
end

#run ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
 
# File 'lib/chef/knife/ec_import.rb', line 108

def run
  set_dest_dir_from_args!
  set_client_config!
  ensure_webui_key_exists!
  set_skip_user_acl!

  warn_on_incorrect_clients_group(dest_dir, 'import')

  # Unlike restore, we do NOT restore users or user SQL data
  # We assume users already exist (managed by IAM)

  for_each_organization do |orgname|
    ui.msg "Verifying organization[#{orgname}]"
    
    # Unlike restore, we do NOT create the organization
    # We validate it exists instead
    unless organization_exists?(orgname)
      ui.error("Organization #{orgname} does not exist. Skipping.")
      next
    end
    
    # Note: We skip importing invitations and adding users to org
    # User org membership is now managed by the platform
    # and invites cannot be acted upon by users
    upload_org_data(orgname)
  end

  # Unlike restore, we do NOT restore key SQL data

  if config[:skip_useracl]
    ui.warn('Skipping user ACL update. To update user ACLs, remove --skip-useracl.')
  else
    restore_user_acls
  end

  completion_banner
end

#set_client_config! ⇒ Object

Override set_client_config! to add Tenant-Id header when provided



89
90
91
92
93
94
 
# File 'lib/chef/knife/ec_import.rb', line 89

def set_client_config!
  super
  if config[:tenant_id_header]
    Chef::Config.custom_http_headers = (Chef::Config.custom_http_headers || {}).merge({'Tenant-Id' => config[:tenant_id_header]})
  end
end

#set_skip_user_acl! ⇒ Object

Override set_skip_user_acl! to avoid calling server.version



97
98
99
100
101
 
# File 'lib/chef/knife/ec_import.rb', line 97

def set_skip_user_acl!
  # Skip user ACLs only if explicitly requested via --skip-useracl flag
  # Default behavior: assume user ACLs are supported
  config[:skip_useracl] ||= false
end

#sort_groups_for_upload(groups) ⇒ Object 



297
298
299
 
# File 'lib/chef/knife/ec_import.rb', line 297

def sort_groups_for_upload(groups)
  Chef::Tsorter.new(group_array_to_sortable_hash(groups)).tsort
end

#upload_org_data(name) ⇒ Object 



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/chef/knife/ec_import.rb', line 196

def upload_org_data(name)
  old_config = Chef::Config.save

  begin
    # Clear out paths
    PATHS.each do |path|
      Chef::Config.delete(path.to_sym)
    end

    Chef::Config.chef_repo_path = "#{dest_dir}/organizations/#{name}"
    Chef::Config.versioned_cookbooks = true
    Chef::Config.chef_server_url = "#{server.root_url}/organizations/#{name}"

    # Upload the admins, public_key_read_access and billing-admins groups and acls
    ui.msg 'Restoring org admin data'
    chef_fs_config = Chef::ChefFS::Config.new

    # Handle Admins, Billing Admins and Public Key Read Access separately
    groups = get_admin_groups(chef_fs_config)

    groups.each do |group|
      restore_group(chef_fs_config, group, :clients => false)
    end

    acls_groups_paths = get_admin_group_acl_paths(chef_fs_config)

    acls_groups_paths.each do |acl|
      chef_fs_copy_pattern(acl, chef_fs_config)
    end

    # For import command, we default to using 'pivotal' as node_name
    # since we assume modern Chef Server (version check is skipped)
    Chef::Config.node_name = config[:skip_version] ? org_admin : 'pivotal'

    # Restore the entire org skipping the admin data and restoring groups and acls last
    # Also skip:
    #   - org.json: organization metadata is not managed by import (org must pre-exist)
    #   - members.json, invitations.json: user-org membership is managed by the platform
    #   - acls, groups: handled separately below in the correct order
    ui.msg 'Restoring the rest of the org'
    chef_fs_config = Chef::ChefFS::Config.new
    skip_entries = %w(acls groups org.json members.json invitations.json)
    top_level_paths = chef_fs_config.local_fs.children.select { |entry| !skip_entries.include?(entry.name) }.map { |entry| entry.path }

    # Topologically sort groups for upload
    unsorted_groups = list_chef_fs_entries(chef_fs_config, '/groups/*', ADMIN_GROUP_FILES)
                        .filter_map do |entry|
                          begin
                            JSON.parse(entry.read)
                          rescue JSON::ParserError, Chef::Exceptions::JSON::ParseError => e
                            ui.warn "Failed to parse group file #{entry.name}: #{e.message}. Skipping."
                            knife_ec_error_handler.add(e)
                            nil
                          end
                        end
    group_paths = sort_groups_for_upload(unsorted_groups).map { |group_name| "/groups/#{group_name}.json" }

    group_acl_paths = list_chef_fs_entries(chef_fs_config, '/acls/groups/*', ADMIN_GROUP_FILES)
                        .map { |entry| entry.path }
    acl_paths = Chef::ChefFS::FileSystem.list(chef_fs_config.local_fs, Chef::ChefFS::FilePattern.new('/acls/*'))
                  .select { |entry| entry.name != 'groups' }
                  .map { |entry| entry.path }

    # Store organization data in a particular order:
    # - clients must be uploaded before groups (in top_level_paths)
    # - groups must be uploaded before any acl's
    # - groups must be uploaded twice to account for Chef Infra Server versions that don't
    #   accept group members on POST
    (top_level_paths + group_paths*2 + group_acl_paths + acl_paths).each do |path|
      chef_fs_copy_pattern(path, chef_fs_config)
    end

    # Apply frozen status to cookbooks after they are uploaded
    restore_cookbook_frozen_status(name, chef_fs_config)

    # restore clients to groups, using the pivotal user again
    Chef::Config[:node_name] = 'pivotal'
    groups.each do |group|
      restore_group(Chef::ChefFS::Config.new, group)
    end
   ensure
    Chef::Config.restore(old_config)
  end
end

#user_acl_rest ⇒ Object

Override user_acl_rest to avoid calling server.version



104
105
106
 
# File 'lib/chef/knife/ec_import.rb', line 104

def user_acl_rest
  @user_acl_rest ||= rest
end