Class: Torch::TorchRun::Launcher

Inherits:
Object
  • Object
show all
Defined in:
lib/torch/torchrun.rb

Instance Method Summary collapse

Constructor Details

#initialize(options, script, script_args, out: $stdout, err: $stderr) ⇒ Launcher

Returns a new instance of Launcher.



159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/torch/torchrun.rb', line 159

def initialize(options, script, script_args, out: $stdout, err: $stderr)
  @options = options
  @script = script
  @script_args = script_args
  @out = out
  @err = err

  @local_world_size = determine_local_world_size(@options[:nproc_per_node])
  @min_nodes, @max_nodes = parse_nnodes(@options[:nnodes])
  @num_nodes = ensure_fixed_nnodes(@min_nodes, @max_nodes)
  @node_rank = @options[:node_rank]
  @max_restarts = [@options[:max_restarts], 0].max
  @monitor_interval = [@options[:monitor_interval], 0.0].max
  @role = @options[:role]
  @pass_local_rank_arg = @options[:pass_local_rank_arg]
  @no_ruby = @options[:no_ruby]
  validate_node_rank!

  setup_rendezvous!
end

Instance Method Details

#run ⇒ Object 



180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'lib/torch/torchrun.rb', line 180

def run
  restarts = 0

  loop do
    status = launch_worker_group(restarts)
    return status if status.zero? || @signal_received
    return status if restarts >= @max_restarts

    restarts += 1
    log("Worker group failed (exit #{status}). Restarting #{restarts}/#{@max_restarts} ...")
    sleep(@monitor_interval) if @monitor_interval.positive?
  end
end