Class: Curl::Multi

Inherits:
Object
  • Object
show all
Defined in:
lib/curb.rb,
ext/curb_multi.c

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.Curl::Multi.default_timeout( = 4) ⇒ 4

Get the global default time out for all Curl::Multi Handles.

Returns:

  • (4) 


121
122
123
 
# File 'ext/curb_multi.c', line 121

VALUE ruby_curl_multi_get_default_timeout(VALUE klass) {
  return INT2FIX(cCurlMutiDefaulttimeout);
}

.Curl::Multi.default_timeout( = 4) ⇒ 4

Set the global default time out for all Curl::Multi Handles. This value is used when libcurl cannot determine a timeout value when calling curl_multi_timeout.

Returns:

  • (4) 


109
110
111
112
 
# File 'ext/curb_multi.c', line 109

VALUE ruby_curl_multi_set_default_timeout(VALUE klass, VALUE timeout) {
  cCurlMutiDefaulttimeout = FIX2LONG(timeout);
  return timeout;
}

.download(urls, easy_options = {}, multi_options = {}, download_paths = nil, &blk) ⇒ Object

call-seq:

Curl::Multi.download(){|c|}

will create 2 new files file1.txt and file2.txt

2 files will be opened, and remain open until the call completes

when using the :post or :put method, urls should be a hash, including the individual post fields per post



202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
 
# File 'lib/curb.rb', line 202

def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
  procs = []
  files = []
  urls_with_config = []
  url_to_download_paths = {}

  urls.each_with_index do|urlcfg,i|
    if urlcfg.is_a?(Hash)
      url = url[:url]
    else
      url = urlcfg
    end

    if download_paths and download_paths[i]
      download_path = download_paths[i]
    else
      download_path = File.basename(url)
    end

    lambda do|dp|
      file = File.open(dp,"wb")
      procs << (lambda {|data| file.write data; data.size })
      files << file
    end.call(download_path)

    if urlcfg.is_a?(Hash)
      urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
    else
      urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
    end
    url_to_download_paths[url] = download_path # store for later
  end

  if blk
    Curl::Multi.http(urls_with_config, multi_options) {|c,code,method| blk.call(c,url_to_download_paths[c.url]) }
  else
    Curl::Multi.http(urls_with_config, multi_options)
  end

ensure
  errors = []
  files.each {|f|
    begin
      f.close
    rescue => e
      errors << e
    end
  }
  raise errors unless errors.empty?
end

.get(urls, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.get('url1','url2','url3','url4','url5', :follow_location => true) do|easy|
  easy
end

Blocking call to fetch multiple url’s in parallel.



67
68
69
70
71
72
73
 
# File 'lib/curb.rb', line 67

def get(urls, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls.each do|url|
    url_confs << {:url => url, :method => :get}.merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
end

.http(urls_with_config, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.http( [

{ :url => 'url1', :method => :post,
  :post_fields => {'field1' => 'value1', 'field2' => 'value2'} },
{ :url => 'url2', :method => :get,
  :follow_location => true, :max_redirects => 3 },
{ :url => 'url3', :method => :put, :put_data => File.open('file.txt','rb') },
{ :url => 'url4', :method => :head }

], => true)

Blocking call to issue multiple HTTP requests with varying verb’s.

urls_with_config: is a hash of url’s pointing to the easy handle options as well as the special option :method, that can by one of [:get, :post, :put, :delete, :head], when no verb is provided e.g. :method => nil -> GET is used multi_options: options for the multi handle blk: a callback, that yeilds when a handle is completed



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/curb.rb', line 141

def http(urls_with_config, multi_options={}, &blk)
  m = Curl::Multi.new
  # configure the multi handle
  multi_options.each { |k,v| m.send("#{k}=", v) }
  callbacks = [:on_progress,:on_debug,:on_failure,:on_success,:on_body,:on_header]

  urls_with_config.each do|conf|
    c = conf.dup # avoid being destructive to input
    url     = c.delete(:url)
    method  = c.delete(:method)
    headers = c.delete(:headers)

    easy    = Curl::Easy.new(url)

    # assign callbacks
    callbacks.each do |cb|
      cbproc = c.delete(cb)
      easy.send(cb,&cbproc) if cbproc
    end

    case method
    when :post
      fields = c.delete(:post_fields)
      # set the post post using the url fields
      easy.post_body = fields.map{|f,k| "#{easy.escape(f)}=#{easy.escape(k)}"}.join('&')
    when :put
      easy.put_data = c.delete(:put_data)
    when :head
      easy.head = true
    when :delete
      easy.delete = true
    when :get
    else
      # XXX: nil is treated like a GET
    end

    # headers is a special key
    headers.each {|k,v| easy.headers[k] = v } if headers
 
    #
    # use the remaining options as specific configuration to the easy handle
    # bad options should raise an undefined method error
    #
    c.each { |k,v| easy.send("#{k}=",v) }

    easy.on_complete {|curl,code| blk.call(curl,code,method) } if blk
    m.add(easy)
  end
  m.perform
end

.Curl::Multi.new ⇒ #<Curl::Easy...>

Create a new Curl::Multi instance

Returns:



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'ext/curb_multi.c', line 81

VALUE ruby_curl_multi_new(VALUE klass) {
  VALUE new_curlm;

  ruby_curl_multi *rbcm = ALLOC(ruby_curl_multi);

  rbcm->handle = curl_multi_init();
  if (!rbcm->handle) {
    rb_raise(mCurlErrFailedInit, "Failed to initialize multi handle");
  }

  rbcm->requests = rb_hash_new();

  rbcm->active = 0;
  rbcm->running = 0;

  new_curlm = Data_Wrap_Struct(klass, curl_multi_mark, curl_multi_free, rbcm);

  return new_curlm;
}

.post(urls_with_config, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.post([{:url => 'url1', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
                  {:url => 'url2', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}},
                  {:url => 'url3', :post_fields => {'field1' => 'value1', 'field2' => 'value2'}}],
                 { :follow_location => true, :multipart_form_post => true },
                 {:pipeline => true }) do|easy|
  easy_handle_on_request_complete
end

Blocking call to POST multiple form’s in parallel.

urls_with_config: is a hash of url’s pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi handle



91
92
93
94
95
96
97
 
# File 'lib/curb.rb', line 91

def post(urls_with_config, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls_with_config.each do|uconf|
    url_confs << uconf.merge(:method => :post).merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
end

.put(urls_with_config, easy_options = {}, multi_options = {}, &blk) ⇒ Object

call-seq:

Curl::Multi.put([{:url => 'url1', :put_data => "some message"},
                 {:url => 'url2', :put_data => IO.read('filepath')},
                 {:url => 'url3', :put_data => "maybe another string or socket?"],
                 {:follow_location => true},
                 {:pipeline => true }) do|easy|
  easy_handle_on_request_complete
end

Blocking call to POST multiple form’s in parallel.

urls_with_config: is a hash of url’s pointing to the postfields to send easy_options: are a set of common options to set on all easy handles multi_options: options to set on the Curl::Multi handle



115
116
117
118
119
120
121
 
# File 'lib/curb.rb', line 115

def put(urls_with_config, easy_options={}, multi_options={}, &blk)
  url_confs = []
  urls_with_config.each do|uconf|
    url_confs << uconf.merge(:method => :put).merge(easy_options)
  end
  self.http(url_confs, multi_options) {|c,code,method| blk.call(c) }
end

Instance Method Details

#add(easy) ⇒ Object

multi = Curl::Multi.new easy = Curl::Easy.new(‘url’)

multi.add(easy)

Add an easy handle to the multi stack



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
 
# File 'ext/curb_multi.c', line 219

VALUE ruby_curl_multi_add(VALUE self, VALUE easy) {
  CURLMcode mcode;
  ruby_curl_easy *rbce;
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  Data_Get_Struct(easy, ruby_curl_easy, rbce);

  /* setup the easy handle */
  ruby_curl_easy_setup( rbce );

  mcode = curl_multi_add_handle(rbcm->handle, rbce->curl);
  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
    raise_curl_multi_error_exception(mcode);
  }

  rbcm->active++;

  /* Increase the running count, so that the perform loop keeps running.
   * If this number is not correct, the next call to curl_multi_perform will correct it. */
  rbcm->running++;

  rb_hash_aset( rbcm->requests, easy, easy );

  rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );

  return self;
}

#cancel! ⇒ Object

Cancels all requests currently being made on this Curl::Multi handle.



310
311
312
313
314
315
316
317
318
319
 
# File 'ext/curb_multi.c', line 310

static VALUE ruby_curl_multi_cancel(VALUE self) {
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  
  rb_hash_foreach( rbcm->requests, ruby_curl_multi_cancel_callback, (VALUE)rbcm );
  
  /* for chaining */
  return self;
}

#idle? ⇒ Boolean

Returns whether or not this Curl::Multi handle is processing any requests. E.g. this returns true when multi.requests.length == 0.

Returns:

  • (Boolean) 


159
160
161
162
163
164
165
166
167
168
169
 
# File 'ext/curb_multi.c', line 159

static VALUE ruby_curl_multi_idle(VALUE self) {
  ruby_curl_multi *rbcm;
  
  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  
  if ( FIX2INT( rb_funcall(rbcm->requests, rb_intern("length"), 0) ) == 0 ) {
    return Qtrue;
  } else {
    return Qfalse;
  }
}

#max_connects=(count) ⇒ Object

multi = Curl::Multi.new multi.max_connects = 800

Set the max connections in the cache for a multi handle



178
179
180
181
182
183
184
185
186
187
 
# File 'ext/curb_multi.c', line 178

static VALUE ruby_curl_multi_max_connects(VALUE self, VALUE count) {
#ifdef HAVE_CURLMOPT_MAXCONNECTS
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  curl_multi_setopt(rbcm->handle, CURLMOPT_MAXCONNECTS, NUM2INT(count));
#endif

  return count;
}

#perform(*args) ⇒ Object

multi = Curl::Multi.new easy1 = Curl::Easy.new(‘url’) easy2 = Curl::Easy.new(‘url’)

multi.add(easy1) multi.add(easy2)

multi.perform do

# while idle other code my execute here

end

Run multi handles, looping selecting when data can be transfered



414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
 
# File 'ext/curb_multi.c', line 414

VALUE ruby_curl_multi_perform(int argc, VALUE *argv, VALUE self) {
  CURLMcode mcode;
  ruby_curl_multi *rbcm;
  int maxfd, rc;
  fd_set fdread, fdwrite, fdexcep;

  long timeout_milliseconds;
  struct timeval tv = {0, 0};
  VALUE block = Qnil;

  rb_scan_args(argc, argv, "0&", &block);

  Data_Get_Struct(self, ruby_curl_multi, rbcm);

  timeout_milliseconds = cCurlMutiDefaulttimeout;

  rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
 
  do {
    while (rbcm->running) {

#ifdef HAVE_CURL_MULTI_TIMEOUT
      /* get the curl suggested time out */
      mcode = curl_multi_timeout(rbcm->handle, &timeout_milliseconds);
      if (mcode != CURLM_OK) {
        raise_curl_multi_error_exception(mcode);
      }
#else
      /* libcurl doesn't have a timeout method defined, initialize to -1 we'll pick up the default later */
      timeout_milliseconds = -1;
#endif

      if (timeout_milliseconds == 0) { /* no delay */
        rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
        continue;
      }
      else if (timeout_milliseconds < 0) {
        timeout_milliseconds = cCurlMutiDefaulttimeout; /* libcurl doesn't know how long to wait, use a default timeout */
      }

      if (timeout_milliseconds > cCurlMutiDefaulttimeout) {
        timeout_milliseconds = cCurlMutiDefaulttimeout; /* buggy versions libcurl sometimes reports huge timeouts... let's cap it */
      }

      tv.tv_sec  = 0; /* never wait longer than 1 second */
      tv.tv_usec = timeout_milliseconds * 1000;

      if (timeout_milliseconds == 0) { /* no delay */
        rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
        continue;
      }

      if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }

      FD_ZERO(&fdread);
      FD_ZERO(&fdwrite);
      FD_ZERO(&fdexcep);
      /* load the fd sets from the multi handle */
      mcode = curl_multi_fdset(rbcm->handle, &fdread, &fdwrite, &fdexcep, &maxfd);
      if (mcode != CURLM_OK) {
        raise_curl_multi_error_exception(mcode);
      }

      rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
      switch(rc) {
      case -1:
        rb_raise(rb_eRuntimeError, "select(): %s", strerror(errno));
        break;
      case 0:
        rb_curl_multi_read_info( self, rbcm->handle );
        if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
      default: 
        rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
        break;
      }
    }
    rb_curl_multi_read_info( self, rbcm->handle );
    if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
  } while( rbcm->running );
    
  return Qtrue;
}

#pipeline=(onoff) ⇒ Object

multi = Curl::Multi.new multi.pipeline = true

Pass a long set to 1 to enable or 0 to disable. Enabling pipelining on a multi handle will make it attempt to perform HTTP Pipelining as far as possible for transfers using this handle. This means that if you add a second request that can use an already existing connection, the second request will be “piped” on the same connection rather than being executed in parallel. (Added in 7.16.0)



200
201
202
203
204
205
206
207
208
 
# File 'ext/curb_multi.c', line 200

static VALUE ruby_curl_multi_pipeline(VALUE self, VALUE onoff) {
#ifdef HAVE_CURLMOPT_PIPELINING
  ruby_curl_multi *rbcm;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  curl_multi_setopt(rbcm->handle, CURLMOPT_PIPELINING, onoff == Qtrue ? 1 : 0);
#endif
  return onoff;
}

#remove(easy) ⇒ Object

multi = Curl::Multi.new easy = Curl::Easy.new(‘url’)

multi.add(easy)

# sometime later multi.remove(easy)

Remove an easy handle from a multi stack.

Will raise an exception if the easy handle is not found



262
263
264
265
266
267
268
269
270
271
272
273
 
# File 'ext/curb_multi.c', line 262

VALUE ruby_curl_multi_remove(VALUE self, VALUE easy) {
  ruby_curl_multi *rbcm;
  ruby_curl_easy *rbce;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);

  Data_Get_Struct(easy, ruby_curl_easy, rbce);

  rb_curl_multi_remove(rbcm,easy);

  return self;
}

#requests ⇒ Array

Returns an array containing all the active requests on this Curl::Multi object.

Returns:

  • (Array) 


138
139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'ext/curb_multi.c', line 138

static VALUE ruby_curl_multi_requests(VALUE self) {
  ruby_curl_multi *rbcm;
  VALUE result_array;

  Data_Get_Struct(self, ruby_curl_multi, rbcm);
  
  result_array = rb_ary_new();
  
  /* iterate over the requests hash, and stuff references into the array. */
  rb_hash_foreach(rbcm->requests, ruby_curl_multi_requests_callback, result_array);
  
  return result_array;
}