Module: Aspera::Sync::Operations

Defined in:
lib/aspera/sync/operations.rb

Overview

builds command line arg for async and execute it

Constant Summary collapse

DIRECTIONS =

Sync direction

%i[push pull bidi].freeze
DEFAULT_DIRECTION =

Default direction for sync

DIRECTIONS.first
SCP_REMOTE_REGEX = 
/\A(?:(?:(?<user>[^@:\s]+)@)?(?<host>[^:\s]+):)?(?<path>.+)\z/
CONF_SCHEMA = 
CommandLineBuilder.read_schema(Schema::Registry::SYNC_CONF)

Class Method Summary collapse

Class Method Details

.admin_status(sync_info) ⇒ Hash

Run ‘asyncadmin` to get status of sync session

Parameters:

  • sync_info (Hash) —

    sync parameters in conf or args format

Returns:

  • (Hash) —

    parsed output of asyncadmin



179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
 
# File 'lib/aspera/sync/operations.rb', line 179

def admin_status(sync_info)
  Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
  arguments = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
  if sync_info.key?('local')
    # `conf` format
    arguments.push("--name=#{sync_info['name']}")
    if sync_info.key?('local_db_dir')
      arguments.push("--local-db-dir=#{sync_info['local_db_dir']}")
    elsif sync_info.dig('local', 'path')
      arguments.push("--local-dir=#{sync_info.dig('local', 'path')}")
    else
      raise Error, 'Missing either local_db_dir or local.path'
    end
  else
    # `args` format
    session = sync_info['sessions'].first
    arguments.push("--name=#{session['name']}")
    if session.key?('local_db_dir')
      arguments.push("--local-db-dir=#{session['local_db_dir']}")
    elsif session.key?('local_dir')
      arguments.push("--local-dir=#{session['local_dir']}")
    else
      raise Error, 'Missing either local_db_dir or local_dir'
    end
  end
  stdout = Environment.secure_execute(*arguments, mode: :capture).first
  return parse_status(stdout)
end

.args_to_conf(args) ⇒ Object

Translate ‘async` native command line arguments to `conf` JSON



304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
 
# File 'lib/aspera/sync/operations.rb', line 304

def args_to_conf(args)
  result = {}
  while args.any?
    option = args.shift
    if option =~ /^(--[^=]+)=(.*)$/
      option = ::Regexp.last_match(1) # "--toto"
      args.unshift(::Regexp.last_match(2))
    end
    if option.eql?('--preserve-time') || option.eql?('-t')
      args.unshift('--preserve-creation-time') if Environment.instance.os.eql?(Environment::OS_WINDOWS)
      option = '--preserve-modification-time'
    end
    if option.eql?('--remote') || option.eql?('-r')
      value = args.first
      if (m = SCP_REMOTE_REGEX.match(value))
        if m[:host]
          args.shift
          args.unshift("--host=#{m[:host]}")
          args.unshift("--user=#{m[:user]}") if m[:user]
          args.unshift(m[:path])
        end
      end
    end
    path, props = find_option(CONF_SCHEMA, [], option)
    raise "Option not found: #{option}" if path.nil?
    last_key = path.pop
    # navigate in the current result to insert the value
    current = result
    path.each do |key|
      current[key] ||= {}
      current = current[key]
    end
    current[last_key] = props['x-cli-switch'] ? true : args.shift
  end
  return result
end

.direction_sym(sync_info) ⇒ Symbol

Get symbol of sync direction, defaulting to :push

Parameters:

  • sync_info (Hash) —

    Sync parameters, ‘conf` or `args` format

Returns:

  • (Symbol) —

    direction symbol, one of :push, :pull, :bidi



78
79
80
 
# File 'lib/aspera/sync/operations.rb', line 78

def direction_sym(sync_info)
  (sync_info['direction'] || DEFAULT_DIRECTION).to_sym
end

.find_option(schema, path, option) ⇒ Array?

Search given option in JSON Schema tree

Parameters:

  • schema (Hash) —

    JSON Schema tree (root or sub-tree)

  • path (Array) —

    Path to subtree

  • option (String) —

    Option to search

Returns:

  • (Array, nil) —

    with path/schema for that option



289
290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'lib/aspera/sync/operations.rb', line 289

def find_option(schema, path, option)
  if %w[x-cli-option x-cli-short].any?{ |i| schema[i].eql?(option)}
    Log.log.debug('Special') if schema['x-cli-special']
    return [path, schema]
  end
  if schema['type'].eql?('object')
    schema['properties']&.each do |name, props|
      res = find_option(props, path + [name], option)
      return res unless res.nil?
    end
  end
  return
end

.list_db_files(local_db_dir) ⇒ Object 



253
254
255
256
257
258
259
 
# File 'lib/aspera/sync/operations.rb', line 253

def list_db_files(local_db_dir)
  private = File.join(local_db_dir, PRIVATE_FOLDER)
  Dir.children(private).filter_map do |name|
    db_file = File.join(private, name, ASYNC_DB)
    [name, db_file] if File.exist?(db_file)
  end.to_h
end

.local_db_folder(sync_info) ⇒ String?

Find the local database folder based on sync_info

Parameters:

  • sync_info (Hash) —

    sync parameters in conf or args format

Returns:

  • (String, nil) —

    Path to “local DB dir”, i.e. folder that contains folders that contain ‘snap.db`



211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/aspera/sync/operations.rb', line 211

def local_db_folder(sync_info)
  Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
  if sync_info.key?('local')
    # `conf` format
    if sync_info.key?('local_db_dir')
      return sync_info['local_db_dir']
    elsif (local_path = sync_info.dig('local', 'path'))
      return local_path
    elsif exception
      raise Error, 'Missing either local_db_dir or local.path'
    end
  else
    # `args` format
    session = sync_info['sessions'].first
    if session.key?('local_db_dir')
      return session['local_db_dir']
    elsif session.key?('local_dir')
      return session['local_dir']
    elsif exception
      raise Error, 'Missing either local_db_dir or local_dir'
    end
  end
  nil
end

.parse_status(stdout) ⇒ Object

Parse output of asyncadmin



159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/aspera/sync/operations.rb', line 159

def parse_status(stdout)
  Log.log.trace1{"stdout=#{stdout}"}
  result = {}
  ids = nil
  stdout.split("\n").each do |line|
    info = line.split(':', 2).map(&:lstrip)
    if info[1].eql?('')
      info[1] = ids = []
    elsif info[1].nil?
      ids.push(info[0])
      next
    end
    result[info[0]] = info[1]
  end
  return result
end

.remote_certificates(remote) ⇒ Array<String>

Get certificates to use for remote connection

Parameters:

  • remote (Hash) —

    remote connection parameters

Returns:

  • (Array<String>) —

    list of certificate file paths



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/aspera/sync/operations.rb', line 51

def remote_certificates(remote)
  certificates_to_use = []
  # use web socket secure for session ?
  if remote['connect_mode']&.eql?('ws')
    remote.delete('port')
    remote.delete('fingerprint')
    # ignore cert for wss ?
    # if @options[:check_ignore_cb]&.call(remote['host'], remote['ws_port'])
    #   wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
    #   wss_url = "https://#{remote['host']}:#{remote['ws_port']}"
    #   File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
    #   certificates_to_use.push(wss_cert_file)
    # end
    # set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
    # certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs]
  else
    # remove unused parameter (avoid warning)
    remote.delete('ws_port')
    # add SSH bypass keys when authentication is token and no auth is provided
    certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa)) if remote.key?('token') && !remote.key?('pass')
  end
  return certificates_to_use
end

.session_db_file(sync_info) ⇒ Object 



247
248
249
250
251
 
# File 'lib/aspera/sync/operations.rb', line 247

def session_db_file(sync_info)
  db_file = File.join(local_db_folder(sync_info), PRIVATE_FOLDER, session_name(sync_info), ASYNC_DB)
  Aspera.assert(File.exist?(db_file)){"Database file #{db_file} does not exist"}
  db_file
end

.session_name(sync_info) ⇒ Object 



236
237
238
239
240
241
242
243
244
245
 
# File 'lib/aspera/sync/operations.rb', line 236

def session_name(sync_info)
  Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
  if sync_info.key?('local')
    # `conf` format
    return sync_info['name']
  else
    # `args` format
    return sync_info['sessions'].first['name']
  end
end

.start(sync_info, opt_ts = nil) {|direction, local_dir, remote_dir| ... } ⇒ Object

Start the sync process

Parameters:

  • sync_info (Hash) —

    Sync parameters, old or new format

  • opt_ts (Hash) (defaults to: nil) —

    Optional transfer spec

Yield Parameters:

  • direction (Symbol) —

    Sync direction (one of DIRECTIONS: :push, :pull, :bidi)

  • local_dir (String) —

    Local directory path

  • remote_dir (String) —

    Remote directory path

Yield Returns:

  • (Hash) —

    Transfer spec to use for authentication



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/aspera/sync/operations.rb', line 89

def start(sync_info, opt_ts = nil)
  Log.dump(:sync_params_initial, sync_info)
  Aspera.assert_type(sync_info, Hash)
  Aspera.assert(PARAM_KEYS.any?{ |k| sync_info.key?(k)}, type: Error){'At least one of `local` or `sessions` must be present in async parameters'}
  env_args = {
    args: [],
    env:  {}
  }
  if sync_info.key?('local')
    # `conf` format
    Aspera.assert_type(sync_info['local'], Hash){'local'}
    remote = sync_info['remote']
    Aspera.assert_type(remote, Hash){'remote'}
    Aspera.assert_type(remote['path'], String){'remote path'}
    # get transfer spec if possible, and feed back to new structure
    if block_given?
      transfer_spec = yield(direction_sym(sync_info), sync_info['local']['path'], remote['path'])
      Log.dump(:auth_ts, transfer_spec)
      transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
      tspec_to_sync_info(transfer_spec, sync_info, CONF_SCHEMA)
      update_remote_dir(remote, 'path', transfer_spec)
    end
    remote['connect_mode'] ||= transfer_spec['wss_enabled'] ? 'ws' : 'ssh'
    add_certificates = remote_certificates(remote)
    if !add_certificates.empty?
      remote['private_key_paths'] ||= []
      remote['private_key_paths'].concat(add_certificates)
    end
    # '--exclusive-mgmt-port=12345', '--arg-err-path=-',
    env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_info))}"]
    Log.dump(:sync_conf, sync_info)
    agent = Agent::Direct.new
    agent.start_and_monitor_process(session: {}, name: :async, **env_args)
  else
    # `args` format
    raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
      sync_info.keys.push('instance').uniq.sort.eql?(CMDLINE_PARAMS_KEYS)
    Aspera.assert_type(sync_info['sessions'], Array)
    Aspera.assert_type(sync_info['sessions'].first, Hash)
    if block_given?
      sync_info['sessions'].each do |session|
        Aspera.assert_type(session['local_dir'], String){'local_dir'}
        Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
        transfer_spec = yield(direction_sym(session), session['local_dir'], session['remote_dir'])
        Log.dump(:auth_ts, transfer_spec)
        transfer_spec.deep_merge!(opt_ts) unless opt_ts.nil?
        tspec_to_sync_info(transfer_spec, session, ARGS_SESSION_SCHEMA)
        session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa) if transfer_spec.key?('token')
        update_remote_dir(session, 'remote_dir', transfer_spec)
      end
    end
    if sync_info.key?('instance')
      Aspera.assert_type(sync_info['instance'], Hash)
      instance_builder = CommandLineBuilder.new(sync_info['instance'], ARGS_INSTANCE_SCHEMA, CommandLineConverter)
      instance_builder.process_params
      instance_builder.add_env_args(env_args)
    end
    sync_info['sessions'].each do |session_params|
      Aspera.assert_type(session_params, Hash)
      Aspera.assert(session_params.key?('name')){'session must contain at least: name'}
      session_builder = CommandLineBuilder.new(session_params, ARGS_SESSION_SCHEMA, CommandLineConverter)
      session_builder.process_params
      session_builder.add_env_args(env_args)
    end
    Environment.secure_execute(Ascp::Installation.instance.path(:async), *env_args[:args], env: env_args[:env])
  end
  return
end

.tspec_to_sync_info(transfer_spec, sync_info, schema) ⇒ Object

Transfer specification to synchronization information tag ‘x-ts-name` in schema is used to map transfer spec parameters to async `sync_info`

Parameters:

  • transfer_spec (Hash) —

    transfer specification

  • sync_info (Hash) —

    synchronization information

  • schema (Hash) —

    schema definition



268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
 
# File 'lib/aspera/sync/operations.rb', line 268

def tspec_to_sync_info(transfer_spec, sync_info, schema)
  Log.dump(:tspec_to_sync_info, transfer_spec)
  schema['properties'].each do |name, property|
    if property.key?('x-ts-name')
      tspec_param = property['x-ts-name']
      if transfer_spec.key?(tspec_param) && !sync_info.key?(name)
        sync_info[name] = property['x-ts-convert'] ? CommandLineConverter.send(property['x-ts-convert'], transfer_spec[tspec_param]) : transfer_spec[tspec_param]
      end
    end
    if property['type'].eql?('object') && property.key?('properties')
      sync_info[name] ||= {}
      tspec_to_sync_info(transfer_spec, sync_info[name], property)
    end
  end
end

.update_remote_dir(sync_info, remote_dir_key, transfer_spec) ⇒ Object

Set ‘remote_dir` in sync parameters based on transfer spec

Parameters:

  • sync_info (Hash) —

    Sync parameters, in ‘conf` or `args` format.

  • remote_dir_key (String) —

    Key to update in above hash

  • transfer_spec (Hash) —

    Transfer spec



34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/aspera/sync/operations.rb', line 34

def update_remote_dir(sync_info, remote_dir_key, transfer_spec)
  if transfer_spec.dig(*%w[tags aspera node file_id])
    # in AoC, use gen4
    sync_info[remote_dir_key] = '/'
  elsif transfer_spec['cookie']&.start_with?('aspera.shares2')
    # TODO : something more generic, independent of Shares
    # in Shares, the actual folder on remote end is not always the same as the name of the share
    remote_key = transfer_spec['direction'].eql?('send') ? 'destination' : 'source'
    actual_remote = transfer_spec['paths']&.first&.[](remote_key)
    sync_info[remote_dir_key] = actual_remote if actual_remote
  end
  nil
end