Class: Curl::Multi

Inherits:
Object
  • Object
show all
Defined in:
lib/curl/multi.rb,
ext/curb_multi.c

Defined Under Namespace

Classes: DownloadError

Constant Summary collapse

IDLE_EASY_REFERENCES_USE_WEAK_MAP = 
begin
  probe = ObjectSpace::WeakMap.new
  probe[Object.new.freeze] = true
  true
rescue ArgumentError, FrozenError, NameError
  false
end

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Object

Instance methods



546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
 
# File 'ext/curb_multi.c', line 546

static VALUE ruby_curl_multi_initialize(VALUE self) {
  ruby_curl_multi *rbcm;

  TypedData_Get_Struct(self, ruby_curl_multi, &ruby_curl_multi_data_type, rbcm);

  ruby_curl_multi_init(rbcm);

  /*
   * The mark routine will be called by the garbage collector during its ``mark'' phase.
   * If your structure references other Ruby objects, then your mark function needs to
   * identify these objects using rb_gc_mark(value). If the structure doesn't reference
   * other Ruby objects, you can simply pass 0 as a function pointer.
   */
  return self;
}

Class Method Details

.Curl::Multi.autoclose ⇒ Object

Get the global default autoclose setting for all Curl::Multi Handles.



606
607
608
 
# File 'ext/curb_multi.c', line 606

VALUE ruby_curl_multi_get_autoclose(VALUE klass) {
  return cCurlMutiAutoClose == 1 ? Qtrue : Qfalse;
}

.Curl::Multi.autoclose( = true) ⇒ true

Automatically close open connections after each request. Otherwise, the connection will remain open for reuse until the next GC

Returns:

  • (true) 


594
595
596
597
 
# File 'ext/curb_multi.c', line 594

VALUE ruby_curl_multi_set_autoclose(VALUE klass, VALUE onoff) {
  cCurlMutiAutoClose = ((onoff == Qtrue) ? 1 : 0);
  return onoff;
}

.Curl::Multi.default_timeout( = 4) ⇒ 4

Get the global default time out for all Curl::Multi Handles.

Returns:

  • (4) 


582
583
584
 
# File 'ext/curb_multi.c', line 582

VALUE ruby_curl_multi_get_default_timeout(VALUE klass) {
  return LONG2NUM(cCurlMutiDefaulttimeout);
}

.Curl::Multi.default_timeout( = 4) ⇒ 4

Set the global default time out for all Curl::Multi Handles. This value is used when libcurl cannot determine a timeout value when calling curl_multi_timeout.

Returns:

  • (4) 


570
571
572
573
 
# File 'ext/curb_multi.c', line 570

VALUE ruby_curl_multi_set_default_timeout(VALUE klass, VALUE timeout) {
  cCurlMutiDefaulttimeout = NUM2LONG(timeout);
  return timeout;
}

.download(urls, easy_options = {}, multi_options = {}, download_paths = nil, download_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}

will create 2 new files file1.txt and file2.txt, unless either file already exists. Auto-derived filenames are safely derived from the last URL path component. Pass :download_dir as the fifth argument to treat download_paths as basenames inside a trusted directory and reject absolute, parent-directory, dotfile, and nested names.

2 files will be opened, and remain open until the call completes

when using the :post or :put method, urls should be a hash, including the individual post fields per post



212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
 
# File 'lib/curl/multi.rb', line 212

def download(urls,easy_options={},multi_options={},download_paths=nil,download_options={},&blk)
  errors = []
  procs = []
  files = []
  urls_with_config = []
  seen_download_paths = {}
  download_infos = []

  if Curl.download_options_hash?(download_paths) && download_options.empty?
    download_options = download_paths
    download_paths = nil
  end

  urls.each_with_index do|urlcfg,i|
    if urlcfg.is_a?(Hash)
      url = urlcfg[:url]
    else
      url = urlcfg
    end

    download_path_arg = download_paths && download_paths[i]
    download_path, file, safe_output, overwrite = Curl.resolve_download_output(url, download_path_arg, download_options)

    if safe_output
      expanded_path = File.expand_path(download_path)
      raise ArgumentError, "duplicate download destination: #{download_path}" if seen_download_paths[expanded_path]

      seen_download_paths[expanded_path] = true
    end

    download_infos << {
      :url => url,
      :urlcfg => urlcfg,
      :path => download_path,
      :file => file,
      :safe_output => safe_output,
      :overwrite => overwrite
    }
  end

  download_infos.each do |info|
    info[:file] ||= Curl.open_safe_download_output(info[:path], :overwrite => info[:overwrite])
    file = info[:file]
    procs << (lambda {|data| file.write data; data.size })
    files << file

    if info[:urlcfg].is_a?(Hash)
      urls_with_config << info[:urlcfg].merge({:on_body => procs.last, :__curb_internal_info => info}.merge(easy_options))
    else
      urls_with_config << {:url => info[:url], :on_body => procs.last, :method => :get, :__curb_internal_info => info}.merge(easy_options)
    end
  end

  finalize_download = lambda do |curl, info|
    file = info[:file]
    files.reject!{|f| f == file }

    if curl.last_result != 0
      begin
        if info[:safe_output]
          file.close(false)
        else
          file.close
        end
      rescue => e
        errors << e
      end
      err_class, err_summary = Curl::Easy.error(curl.last_result)
      err_detail = curl.last_error
      errors << err_class.new([err_summary, err_detail].compact.join(": "))
      false
    else
      begin
        if info[:safe_output]
          file.close(true)
        else
          file.close
        end
        true
      rescue => e
        errors << e
        false
      end
    end
  end

  Curl::Multi.http(urls_with_config, multi_options) do |c,code,method,info|
    if finalize_download.call(c, info) && blk
      blk.call(c,info[:path])
    end
  end

ensure
  pending_exception = $!
  files.each {|f|
    begin
      if f.is_a?(Curl::SafeDownloadOutput)
        f.close(false)
      else
        f.close
      end
    rescue => e
      errors << e
    end
  }
  if errors.any? && !pending_exception
    de = Curl::Multi::DownloadError.new
    de.errors = errors
    raise de
  end
end

.get(urls, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.get(['url1','url2','url3','url4','url5'], :follow_location => true) do|easy|
easy
end

Blocking call to fetch multiple url's in parallel.



24
25
26
27
28
29
30
 
# File 'lib/curl/multi.rb', line 24

def get(urls, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls.each do|url|
    url_confs << {:url => url, :method => :get}.merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) if blk }
end

.http(urls_with_config, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.http( [ { :url => 'url1', :method => :post, :post_fields => => 'value1', 'field2' => 'value2' }, { :url => 'url2', :method => :get, :follow_location => true, :max_redirects => 3 }, { :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') }, { :url => 'url4', :method => :head } ], => Curl::CURLPIPE_HTTP1)

Blocking call to issue multiple HTTP requests with varying verb's.

urls_with_config: is a hash of url's pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used multi_options: options for the multi handle blk: a callback, that yeilds when a handle is completed



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
 
# File 'lib/curl/multi.rb', line 98

def http(urls_with_config, multi_options={}, &blk)
  m = Curl::Multi.new

  # maintain a sane number of easy handles
  multi_options[:max_connects] = max_connects = multi_options.key?(:max_connects) ? multi_options[:max_connects] : 10

  free_handles = [] # keep a list of free easy handles

  # configure the multi handle
  multi_options.each { |k,v| m.send("#{k}=", v) }
  callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_redirect,:on_missing,:on_body,:on_header]

  add_free_handle = proc do|conf, easy|
    c       = conf.dup # avoid being destructive to input
    url     = c.delete(:url)
    method  = c.delete(:method)
    headers = c.delete(:headers)
    internal_info = c.delete(:__curb_internal_info)

    easy    = Curl::Easy.new if easy.nil?

    easy.url = url

    # assign callbacks
    callbacks.each do |cb|
      cbproc = c.delete(cb)
      easy.send(cb,&cbproc) if cbproc
    end

    case method
    when :post
      fields = c.delete(:post_fields)
      # set the post post using the url fields
      easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
    when :put
      easy.put_data = c.delete(:put_data)
    when :head
      easy.head = true
    when :delete
      easy.delete = true
    when :get
    else
      # XXX: nil is treated like a GET
    end

    # headers is a special key
    headers.each {|k,v| easy.headers[k] = v } if headers

    #
    # use the remaining options as specific configuration to the easy handle
    # bad options should raise an undefined method error
    #
    c.each { |k,v| easy.send("#{k}=",v) }

    easy.on_complete {|curl|
      free_handles << curl
      if blk
        if internal_info
          blk.call(curl,curl.response_code,method,internal_info)
        else
          blk.call(curl,curl.response_code,method)
        end
      end
    }
    m.add(easy)
  end

  max_connects.times do
    conf = urls_with_config.pop
    add_free_handle.call(conf, nil) if conf
    break if urls_with_config.empty?
  end

  consume_free_handles = proc do
    # as we idle consume free handles
    if urls_with_config.size > 0 && free_handles.size > 0
      easy = free_handles.pop
      conf = urls_with_config.pop
      add_free_handle.call(conf, easy) if conf
    end
  end

  begin
    if urls_with_config.empty?
      m.perform
    else
      until urls_with_config.empty?
        m.perform do
          consume_free_handles.call
        end
        consume_free_handles.call
      end
      free_handles = nil
    end
  ensure
    m.close
  end

end

.post(urls_with_config, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
                {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
                {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
               { :follow_location => true, :multipart_form_post => true },
               {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy|
easy_handle_on_request_complete
end

Blocking call to POST multiple form's in parallel.

urls_with_config: is a hash of url's pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi handle



48
49
50
51
52
53
54
 
# File 'lib/curl/multi.rb', line 48

def post(urls_with_config, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls_with_config.each do|uconf|
    url_confs << uconf.merge(:method => :post).merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
end

.put(urls_with_config, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.put([{:url => 'url1', :put_data => "some message"},
               {:url => 'url2', :put_data => IO.read('filepath')},
               {:url => 'url3', :put_data => "maybe another string or socket?"],
               {:follow_location => true},
               {:pipeline => Curl::CURLPIPE_HTTP1}) do|easy|
easy_handle_on_request_complete
end

Blocking call to POST multiple form's in parallel.

urls_with_config: is a hash of url's pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi handle



72
73
74
75
76
77
78
 
# File 'lib/curl/multi.rb', line 72

def put(urls_with_config, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls_with_config.each do|uconf|
    url_confs << uconf.merge(:method => :put).merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
end

Instance Method Details

#_add(easy) ⇒ Object

multi = Curl::Multi.new easy = Curl::Easy.new('url')

multi.add(easy)

Add an easy handle to the multi stack



698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
 
# File 'ext/curb_multi.c', line 698

VALUE ruby_curl_multi_add(VALUE self, VALUE easy) {
  CURLMcode mcode;
  ruby_curl_easy *rbce;
  ruby_curl_multi *rbcm;
  ruby_curl_multi *existing_rbcm;

  TypedData_Get_Struct(self, ruby_curl_multi, &ruby_curl_multi_data_type, rbcm);
  TypedData_Get_Struct(easy, ruby_curl_easy, &ruby_curl_easy_data_type, rbce);
  ruby_curl_multi_ensure_handle(rbcm);

  if (rb_curl_multi_has_easy(rbcm, rbce)) {
    return self;
  }

  existing_rbcm = ruby_curl_multi_pointer_if_compatible(rbce->multi);
  if (existing_rbcm && existing_rbcm != rbcm && rb_curl_multi_has_easy(existing_rbcm, rbce)) {
    rb_raise(rb_eRuntimeError, "Cannot add an active Curl::Easy handle to another Curl::Multi");
  }

  /* setup the easy handle */
  ruby_curl_easy_setup( rbce );

  mcode = curl_multi_add_handle(rbcm->handle, rbce->curl);
  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
    ruby_curl_easy_cleanup(easy, rbce);

    raise_curl_multi_error_exception(mcode);
  }

  rbcm->active++;

  /* Increase the running count, so that the perform loop keeps running.
   * If this number is not correct, the next call to curl_multi_perform will correct it. */
  rbcm->running++;

  if (!rbcm->attached) {
    rbcm->attached = st_init_numtable();
    if (!rbcm->attached) {
      curl_multi_remove_handle(rbcm->handle, rbce->curl);
      ruby_curl_easy_cleanup(easy, rbce);
      rb_raise(rb_eNoMemError, "Failed to allocate multi attachment table");
    }
  }

  rbce->multi_attachment_generation++;
  st_insert(rbcm->attached, (st_data_t)rbce, (st_data_t)easy);

  /* track a reference to associated multi handle */
  rbce->multi = self;

  return self;
}

#_close ⇒ Object

multi.close after closing the multi handle all connections will be closed and the handle will no longer be usable



2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
 
# File 'ext/curb_multi.c', line 2260

VALUE ruby_curl_multi_close(VALUE self) {
  ruby_curl_multi *rbcm;
  TypedData_Get_Struct(self, ruby_curl_multi, &ruby_curl_multi_data_type, rbcm);

  if ((rbcm->perform_active || rbcm->callback_active) && !rbcm->allow_close_during_perform) {
    rb_raise(rb_eRuntimeError, "Cannot close an active Curl::Multi handle during perform");
  }

  rb_curl_multi_detach_all(rbcm);

  if (rbcm->handle) {
    curl_multi_cleanup(rbcm->handle);
    rbcm->handle = NULL;
  }

  rbcm->active = 0;
  rbcm->running = 0;
  clear_multi_deferred_exception_if_any(self);
  clear_multi_deferred_exception_source_id_if_any(self);
  return self;
}

#_curb_native_perform ⇒ Object 



426
 
# File 'lib/curl/multi.rb', line 426

alias_method :_curb_native_perform, :perform

#_remove(rb_easy_handle) ⇒ Object

multi = Curl::Multi.new easy = Curl::Easy.new('url')

multi.add(easy)

sometime later

multi.remove(easy)

Remove an easy handle from a multi stack.

Will raise an exception if the easy handle is not found



765
766
767
768
769
770
771
772
773
 
# File 'ext/curb_multi.c', line 765

VALUE ruby_curl_multi_remove(VALUE self, VALUE rb_easy_handle) {
  ruby_curl_multi *rbcm;

  TypedData_Get_Struct(self, ruby_curl_multi, &ruby_curl_multi_data_type, rbcm);

  rb_curl_multi_remove(rbcm, rb_easy_handle);

  return self;
}

#add(easy) ⇒ Object 



442
443
444
445
446
447
448
449
450
451
452
453
 
# File 'lib/curl/multi.rb', line 442

def add(easy)
  return self if requests[easy.object_id]
  # Once a deferred callback exception is pending, Multi#perform is
  # draining existing transfers only and must not start replacement work.
  return self if instance_variable_defined?(:@__curb_deferred_exception)
  Curl.__send__(:apply_safety!, easy) if Curl.respond_to?(:apply_safety!, true)
  _add(easy)
  __unregister_idle_easy_reference(easy)
  requests[easy.object_id] = easy
  __record_native_safety_signature(easy)
  self
end

#cancel! ⇒ Object 



325
326
327
328
329
 
# File 'lib/curl/multi.rb', line 325

def cancel!
  requests.each do |_,easy|
    remove(easy)
  end
end

#close ⇒ Object 



463
464
465
 
# File 'lib/curl/multi.rb', line 463

def close
  __close(true)
end

#idle? ⇒ Boolean

Returns:

  • (Boolean) 


331
332
333
 
# File 'lib/curl/multi.rb', line 331

def idle?
  requests.empty?
end

#max_connects=(count) ⇒ Object

multi = Curl::Multi.new multi.max_connects = 800

Set the max connections in the cache for a multi handle



623
624
625
626
627
628
629
630
631
632
633
634
 
# File 'ext/curb_multi.c', line 623

static VALUE ruby_curl_multi_max_connects(VALUE self, VALUE count) {
#ifdef HAVE_CURLMOPT_MAXCONNECTS
  ruby_curl_multi *rbcm;

  TypedData_Get_Struct(self, ruby_curl_multi, &ruby_curl_multi_data_type, rbcm);
  ruby_curl_multi_ensure_handle(rbcm);

  curl_multi_setopt(rbcm->handle, CURLMOPT_MAXCONNECTS, NUM2LONG(count));
#endif

  return count;
}

#max_host_connections=(count) ⇒ Object

multi = Curl::Multi.new multi.max_host_connections = 1

Set the max number of connections per host



643
644
645
646
647
648
649
650
651
652
653
654
 
# File 'ext/curb_multi.c', line 643

static VALUE ruby_curl_multi_max_host_connections(VALUE self, VALUE count) {
#ifdef HAVE_CURLMOPT_MAX_HOST_CONNECTIONS
  ruby_curl_multi *rbcm;

  TypedData_Get_Struct(self, ruby_curl_multi, &ruby_curl_multi_data_type, rbcm);
  ruby_curl_multi_ensure_handle(rbcm);

  curl_multi_setopt(rbcm->handle, CURLMOPT_MAX_HOST_CONNECTIONS, NUM2LONG(count));
#endif

  return count;
}

#perform(*args) ⇒ Object

perform uses the legacy fdset loop by default, but routes to the socket-action drive loop when the calling fiber runs under a fiber scheduler so sibling fibers keep running during transfers. The socket-action path is also exposed directly as _socket_perform.



2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
 
# File 'ext/curb_multi.c', line 2231

def perform(*args, &block)
  requests.each_value do |easy|
    Curl.__send__(:apply_safety!, easy) if Curl.respond_to?(:apply_safety!, true)
    signature = __curb_safety_signature_for(easy)
    if __curb_native_safety_signatures[easy.object_id] != signature &&
       easy.respond_to?(:__curb_native_setup!, true)
      easy.__send__(:__curb_native_setup!)
      __curb_native_safety_signatures[easy.object_id] = signature
    end
  end

  _curb_native_perform(*args, &block)
end

#pipeline=(method) ⇒ Object

multi = Curl::Multi.new multi.pipeline = true

Pass a long set to 1 for HTTP/1.1 pipelining, 2 for HTTP/2 multiplexing, or 0 to disable. Enabling pipelining on a multi handle will make it attempt to perform HTTP Pipelining as far as possible for transfers using this handle. This means that if you add a second request that can use an already existing connection, the second request will be "piped" on the same connection rather than being executed in parallel. (Added in 7.16.0, multiplex added in 7.43.0)



668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
 
# File 'ext/curb_multi.c', line 668

static VALUE ruby_curl_multi_pipeline(VALUE self, VALUE method) {
#ifdef HAVE_CURLMOPT_PIPELINING
  ruby_curl_multi *rbcm;

  long value;

  if (method == Qtrue) {
    value = 1;
  } else if (method == Qfalse) {
    value  = 0;
  } else {
    value = NUM2LONG(method);
  } 

  TypedData_Get_Struct(self, ruby_curl_multi, &ruby_curl_multi_data_type, rbcm);
  ruby_curl_multi_ensure_handle(rbcm);
  curl_multi_setopt(rbcm->handle, CURLMOPT_PIPELINING, value);
#endif
  return method == Qtrue ? 1 : 0;
}

#remove(easy) ⇒ Object 



455
456
457
458
459
460
461
 
# File 'lib/curl/multi.rb', line 455

def remove(easy)
  return self if !requests[easy.object_id]
  requests.delete(easy.object_id)
  __curb_native_safety_signatures.delete(easy.object_id)
  _remove(easy)
  self
end

#requests ⇒ Object 



335
336
337
 
# File 'lib/curl/multi.rb', line 335

def requests
  @requests ||= {}
end