Module: Sfdown::CLI

Defined in:
lib/sfdown.rb

Overview

Command-line interface: wires the stages together and drives the status bar.

Class Method Summary collapse

Class Method Details

.count_files(node) ⇒ Object 



899
900
901
 
# File 'lib/sfdown.rb', line 899

def count_files(node)
  node.file? ? 1 : node.children.sum { |c| count_files(c) }
end

.dump_tree(node, depth = 0) ⇒ Object

Debug helper: indented tree listing.



806
807
808
809
 
# File 'lib/sfdown.rb', line 806

def dump_tree(node, depth = 0)
  puts "#{'  ' * depth}#{node.name}#{node.dir? ? '/' : ''}"
  node.children.each { |c| dump_tree(c, depth + 1) }
end

.load_metadata(config, bar) ⇒ Object

Bootstrap: rebuild the tree from a metadata file instead of mapping.



887
888
889
890
891
892
893
894
895
896
897
 
# File 'lib/sfdown.rb', line 887

def load_metadata(config, bar)
  root = Metadata.read(config.input)
  bar.log("Loaded metadata from #{config.input}: #{count_files(root)} files")
  root
rescue Errno::ENOENT
  abort "Error: metadata file not found: #{config.input}"
rescue JSON::ParserError => e
  abort "Error: invalid metadata JSON in #{config.input}: #{e.message}"
rescue Metadata::Error => e
  abort "Error: #{e.message}"
end

.main(argv) ⇒ Object 



903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
 
# File 'lib/sfdown.rb', line 903

def main(argv)
  config = Options.parse(argv)
  http = Http.new(timeout: config.timeout, sleep: config.sleep)
  bar = StatusBar.new
  start = Time.now

  root = config.input ? load_metadata(config, bar) : run_stage1(config, http, bar, start)

  dest = File.join(config.output, config.project)
  if config.metadata
    FileUtils.mkdir_p(dest)
    path = File.join(dest, "metadata.json")
    Metadata.write(root, path)
    bar.log("Wrote metadata to #{path}")
  end

  unless config.no
    downloader = Downloader.new(config.project, http, dest, concurrent: config.concurrent,
                                                            link: config.link, log: bar.method(:log))
    run_stage2(downloader, root, bar, start)
  end

  bar.finish
end

.render_stage2(bar, dl, global_start, stage2_start) ⇒ Object 



825
826
827
828
829
830
831
832
833
834
 
# File 'lib/sfdown.rb', line 825

def render_stage2(bar, dl, global_start, stage2_start)
  now = Time.now
  s2 = now - stage2_start
  # Speed reflects only bytes moved this session; resumed bytes weren't
  # transferred now, so counting them would inflate speed and skew the ETA.
  speed = s2.positive? ? dl.downloaded_bytes / s2 : 0
  remaining = [dl.total_bytes - dl.bytes_done, 0].max
  eta = speed.positive? ? remaining / speed : 0
  bar.update("Fetching #{dl.current}", stage2_line(dl, now - global_start, speed, (now - global_start) + eta))
end

.run_stage1(config, http, bar, start) ⇒ Object

Stage 1: map the tree over the network, logging the summary.



871
872
873
874
875
876
877
878
879
880
881
882
883
884
 
# File 'lib/sfdown.rb', line 871

def run_stage1(config, http, bar, start)
  parser = Parser.new(config.project)
  mapper = Mapper.new(config.project, http, parser, concurrent: config.concurrent, log: bar.method(:log))
  root = mapper.map do |node|
    bar.update("Parsing #{node.path}/", stage1_line(mapper, Time.now - start))
  end

  summary = format("Stage 1: %d folders, %d files, %s total in %s",
                   mapper.folders, mapper.files, Fmt.size(mapper.total_size),
                   Fmt.duration(Time.now - start))
  summary += " (#{mapper.failures} failures)" if mapper.failures.positive?
  bar.log(summary)
  root
end

.run_stage2(downloader, root, bar, global_start) ⇒ Object

Drive stage 2: prepare the tree, download files while a ~1s ticker refreshes the bar, apply folder timestamps, then log the stage summary.



838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
 
# File 'lib/sfdown.rb', line 838

def run_stage2(downloader, root, bar, global_start)
  downloader.prepare(root)
  stage2_start = Time.now
  ticker = Thread.new do
    loop do
      render_stage2(bar, downloader, global_start, stage2_start)
      sleep 1
    end
  end

  downloader.download_all
  ticker.kill
  ticker.join
  render_stage2(bar, downloader, global_start, stage2_start) # final 100% frame
  downloader.apply_folder_times(root)

  s2 = Time.now - stage2_start
  avg = s2.positive? ? downloader.downloaded_bytes / s2 : 0
  summary = format("Stage 2: %d/%d files, %s downloaded in %s (avg %s/s)",
                   downloader.files_done, downloader.total_files,
                   Fmt.size(downloader.downloaded_bytes), Fmt.duration(s2), Fmt.size(avg))
  if downloader.resumed_files.positive?
    summary += format(" (%d resumed, %s)", downloader.resumed_files, Fmt.size(downloader.resumed_bytes))
  end
  if downloader.duped_files.positive?
    summary += format(" (%d duplicates, %s saved)", downloader.duped_files, Fmt.size(downloader.duped_bytes))
  end
  summary += " (#{downloader.failures} failures)" if downloader.failures.positive?
  summary += " (#{downloader.mismatches} checksum mismatches)" if downloader.mismatches.positive?
  bar.log(summary)
end

.stage1_line(mapper, elapsed) ⇒ Object

Stage-1 analytics line: starts with the stage number, ends with elapsed time.



812
813
814
815
 
# File 'lib/sfdown.rb', line 812

def stage1_line(mapper, elapsed)
  format("[1] folders: %d | files: %d | size: %s | elapsed: %s",
         mapper.folders, mapper.files, Fmt.size(mapper.total_size), Fmt.duration(elapsed))
end

.stage2_line(dl, global_elapsed, speed, total_est) ⇒ Object

Stage-2 analytics line: elapsed is the global (both-stage) time, complemented by an estimated total = elapsed + remaining_bytes / speed.



819
820
821
822
823
 
# File 'lib/sfdown.rb', line 819

def stage2_line(dl, global_elapsed, speed, total_est)
  format("[2] files: %d/%d | %s/%s | speed: %s/s | elapsed: %s / ~%s",
         dl.files_done, dl.total_files, Fmt.size(dl.bytes_done), Fmt.size(dl.total_bytes),
         Fmt.size(speed), Fmt.duration(global_elapsed), Fmt.duration(total_est))
end