Module: Pindo::FileDownloader

Defined in:
lib/pindo/module/utils/file_downloader.rb

Overview

文件下载工具支持单文件下载和批量并行下载，带重试机制和超时控制

Constant Summary collapse

DEFAULT_MAX_RETRIES =

默认配置

3
DEFAULT_RETRY_DELAY = 
2
DEFAULT_READ_TIMEOUT = 
30
DEFAULT_OPEN_TIMEOUT = 
10
DEFAULT_MAX_THREADS = 
4

Class Method Summary collapse

Class Method Details

.download(url:, dest_path:, max_retries: DEFAULT_MAX_RETRIES, retry_delay: DEFAULT_RETRY_DELAY, silent: false) ⇒ Boolean

下载单个文件

Examples:

基本用法

success = FileDownloader.download(
  url: 'https://example.com/file.zip',
  dest_path: '/tmp/file.zip'
)
raise "下载失败" unless success

自定义重试

success = FileDownloader.download(
  url: 'https://example.com/large.zip',
  dest_path: '/tmp/large.zip',
  max_retries: 5,
  retry_delay: 3,
  silent: true
)

Parameters:

  • url (String) —

    下载 URL

  • dest_path (String) —

    保存路径

  • max_retries (Integer) (defaults to: DEFAULT_MAX_RETRIES) —

    最大重试次数（默认3）

  • retry_delay (Integer) (defaults to: DEFAULT_RETRY_DELAY) —

    重试间隔秒数（默认2）

  • silent (Boolean) (defaults to: false) —

    静默模式，不输出日志（默认false）

Returns:

  • (Boolean) —

    是否下载成功



47
48
49
50
51
52
53
54
55
56
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 47

def self.download(url:, dest_path:, max_retries: DEFAULT_MAX_RETRIES, retry_delay: DEFAULT_RETRY_DELAY, silent: false)
  result = download_internal(
    url: url,
    dest_path: dest_path,
    max_retries: max_retries,
    retry_delay: retry_delay,
    silent: silent
  )
  result[:success]
end

.download_batch(tasks:, max_threads: DEFAULT_MAX_THREADS, fail_fast: false, silent: false) ⇒ Hash

批量并行下载

Examples:

tasks = [
  { url: 'https://example.com/cert1.cer', dest_path: '/tmp/cert1.cer' },
  { url: 'https://example.com/cert2.cer', dest_path: '/tmp/cert2.cer' },
  { url: 'https://example.com/cert3.cer', dest_path: '/tmp/cert3.cer' }
]

result = FileDownloader.download_batch(
  tasks: tasks,
  max_threads: 3
)

if result[:failed] > 0
  puts "部分文件下载失败 (#{result[:failed]}/#{result[:total]})"
  # 查看失败的任务
  result[:results].each do |r|
    puts "失败: #{r[:error]}" unless r[:success]
  end
end

Parameters:

  • tasks (Array<Hash>) —

    下载任务数组每个任务格式: { url: String, dest_path: String }

  • max_threads (Integer) (defaults to: DEFAULT_MAX_THREADS) —

    最大并行线程数（默认4）

  • fail_fast (Boolean) (defaults to: false) —

    遇到错误立即停止（默认false）

  • silent (Boolean) (defaults to: false) —

    静默模式（默认false）

Returns:

  • (Hash) —

    下载结果统计{

    total: Integer,      # 总任务数
success: Integer,    # 成功数量
failed: Integer,     # 失败数量
results: Array       # 每个任务的详细结果 [{ success: Boolean, error: String, task: Hash }]

    }



94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 94

def self.download_batch(tasks:, max_threads: DEFAULT_MAX_THREADS, fail_fast: false, silent: false)
  return { total: 0, success: 0, failed: 0, results: [] } if tasks.empty?

  log_batch_start(tasks.size, max_threads, silent)

  # 任务队列和结果存储
  task_queue = Queue.new
  result_queue = Queue.new
  stop_queue = Queue.new  # 使用 Queue 实现线程安全的停止标志

  # 填充任务队列
  tasks.each_with_index { |task, index| task_queue << [index, task] }

  # 创建工作线程
  threads = []
  max_threads.times do
    threads << Thread.new do
      loop do
        # 检查是否应该停止（fail_fast 模式）
        should_stop = !stop_queue.empty?
        break if task_queue.empty? || should_stop

        # 获取任务
        begin
          index, task = task_queue.pop(true)
        rescue ThreadError
          break  # 队列为空
        end

        # 执行下载
        result = download_internal(
          url: task[:url],
          dest_path: task[:dest_path],
          silent: true  # 批量下载时静默子任务
        )

        # 保存结果
        result_with_meta = {
          index: index,
          success: result[:success],
          error: result[:error],
          file_size: result[:file_size],
          task: task
        }
        result_queue << result_with_meta

        # 显示进度
        completed = result_queue.size
        status = result[:success] ? '✓' : '✗'
        log_batch_progress(status, completed, tasks.size, task[:dest_path], silent)

        # fail_fast 模式下遇到失败立即停止
        if !result[:success] && fail_fast
          stop_queue << true  # 发送停止信号
        end
      end
    end
  end

  # 等待所有线程完成
  threads.each(&:join)

  # 收集结果（按原始顺序排序）
  results_array = []
  until result_queue.empty?
    results_array << result_queue.pop
  end
  results_array.sort_by! { |r| r[:index] }

  # 移除 index 字段（内部使用）
  results = results_array.map do |r|
    {
      success: r[:success],
      error: r[:error],
      file_size: r[:file_size],
      task: r[:task]
    }
  end

  # 统计结果
  success_count = results.count { |r| r[:success] }
  failed_count = results.size - success_count

  log_batch_complete(success_count, failed_count, silent)

  {
    total: tasks.size,
    success: success_count,
    failed: failed_count,
    results: results
  }
end

.download_file(url, dest_path) ⇒ Integer

执行实际的文件下载

Returns:

  • (Integer) —

    文件大小（字节）



244
245
246
247
248
249
250
251
252
253
254
255
256
257
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 244

def self.download_file(url, dest_path)
  uri_options = {
    read_timeout: DEFAULT_READ_TIMEOUT,
    open_timeout: DEFAULT_OPEN_TIMEOUT,
    redirect: true,
    ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE
  }

  URI.open(url, uri_options) do |remote_file|
    content = remote_file.read
    File.binwrite(dest_path, content)
    content.bytesize
  end
end

.download_internal(url:, dest_path:, max_retries: DEFAULT_MAX_RETRIES, retry_delay: DEFAULT_RETRY_DELAY, silent: false) ⇒ Hash

内部下载方法（返回详细结果）

Returns:

  • (Hash) —

    { success: Boolean, error: String, file_size: Integer }



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 196

def self.download_internal(url:, dest_path:, max_retries: DEFAULT_MAX_RETRIES, retry_delay: DEFAULT_RETRY_DELAY, silent: false)
  # 参数验证
  return { success: false, error: 'URL 不能为空', file_size: nil } if url.nil? || url.empty?
  return { success: false, error: '保存路径不能为空', file_size: nil } if dest_path.nil? || dest_path.empty?

  # 确保目录存在
  FileUtils.mkdir_p(File.dirname(dest_path))

  # 执行下载（带重试）
  attempt = 0
  last_error = nil

  while attempt < max_retries
    attempt += 1

    begin
      file_size = download_file(url, dest_path)

      # 验证文件
      unless File.exist?(dest_path) && File.size(dest_path) > 0
        raise "下载的文件为空或不存在"
      end

      # 成功
      log_success(dest_path, file_size, silent)
      return { success: true, error: nil, file_size: file_size }

    rescue => e
      last_error = e

      # 清理失败的文件
      File.delete(dest_path) if File.exist?(dest_path)

      if attempt < max_retries
        log_retry(attempt, max_retries, e.message, retry_delay, silent)
        sleep(retry_delay)
      else
        log_failure(max_retries, e.message, silent)
      end
    end
  end

  # 所有重试都失败
  { success: false, error: last_error&.message || '未知错误', file_size: nil }
end

.format_size(bytes) ⇒ Object

格式化文件大小



260
261
262
263
264
265
266
267
268
269
270
271
272
273
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 260

def self.format_size(bytes)
  return "0 B" if bytes.nil? || bytes == 0

  units = ['B', 'KB', 'MB', 'GB']
  size = bytes.to_f
  unit_index = 0

  while size >= 1024 && unit_index < units.length - 1
    size /= 1024.0
    unit_index += 1
  end

  "#{size.round(2)} #{units[unit_index]}"
end

.log_batch_complete(success_count, failed_count, silent) ⇒ Object

日志：批量下载完成



309
310
311
312
313
314
315
316
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 309

def self.log_batch_complete(success_count, failed_count, silent)
  return if silent
  if failed_count == 0
    Funlog.instance.fancyinfo_success("批量下载完成: 全部成功 (#{success_count})")
  else
    Funlog.instance.fancyinfo_error("批量下载完成: 成功 #{success_count}, 失败 #{failed_count}")
  end
end

.log_batch_progress(status, completed, total, file_path, silent) ⇒ Object

日志：批量下载进度



302
303
304
305
306
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 302

def self.log_batch_progress(status, completed, total, file_path, silent)
  return if silent
  filename = File.basename(file_path)
  Funlog.instance.fancyinfo_update("#{status} 进度: #{completed}/#{total} - #{filename}")
end

.log_batch_start(total, max_threads, silent) ⇒ Object

日志：批量下载开始



296
297
298
299
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 296

def self.log_batch_start(total, max_threads, silent)
  return if silent
  Funlog.instance.fancyinfo_start("开始批量下载 #{total} 个文件（#{max_threads} 线程并行）...")
end

.log_failure(max_retries, error_msg, silent) ⇒ Object

日志：最终失败



290
291
292
293
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 290

def self.log_failure(max_retries, error_msg, silent)
  return if silent
  Funlog.instance.fancyinfo_error("下载失败，已重试#{max_retries}次: #{error_msg}")
end

.log_retry(attempt, max_retries, error_msg, retry_delay, silent) ⇒ Object

日志：重试



284
285
286
287
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 284

def self.log_retry(attempt, max_retries, error_msg, retry_delay, silent)
  return if silent
  Funlog.instance.fancyinfo_error("下载失败 (第#{attempt}/#{max_retries}次): #{error_msg}，#{retry_delay}秒后重试...")
end

.log_success(dest_path, file_size, silent) ⇒ Object

日志：下载成功



276
277
278
279
280
281
 
# File 'lib/pindo/module/utils/file_downloader.rb', line 276

def self.log_success(dest_path, file_size, silent)
  return if silent
  filename = File.basename(dest_path)
  size_str = format_size(file_size)
  Funlog.instance.fancyinfo_success("下载成功: #{filename} (#{size_str})")
end