public
Created

  • Download Gist
downloader.rb
Ruby
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
# Easy download examples
#
# First example, downloads 10 files from the same server sequencially using the same
# easy handle with a persistent connection
#
# Second example, sends all 10 requests in parallel using 10 easy handles
#
$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'ext')))
$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')))
require 'curb'
 
urls = ['http://www.cnn.com/',
'http://www.cnn.com/video/',
'http://newspulse.cnn.com/',
'http://www.cnn.com/US/',
'http://www.cnn.com/WORLD/',
'http://www.cnn.com/POLITICS/',
'http://www.cnn.com/JUSTICE/',
'http://www.cnn.com/SHOWBIZ/',
'http://www.cnn.com/TECH/',
'http://www.cnn.com/HEALTH/']
 
def cleanup(urls)
urls.each do|url|
filename = url.split(/\?/).first.split(/\//).last
File.unlink(filename) if File.exist?(filename)
end
end
 
 
# first sequential
def fetch_sequential(urls)
easy = Curl::Easy.new
easy.follow_location = true
 
urls.each do|url|
easy.url = url
filename = url.split(/\?/).first.split(/\//).last
print "'#{url}' :"
File.open(filename, 'wb') do|f|
easy.on_progress {|dl_total, dl_now, ul_total, ul_now| print "="; true }
easy.on_body {|data| f << data; data.size }
easy.perform
puts "=> '#{filename}'"
end
end
end
 
 
# using multi interface
def fetch_parallel(urls)
Curl::Multi.download(urls){|c,code,method|
filename = c.url.split(/\?/).first.split(/\//).last
puts filename
}
end
 
 
fetch_sequential(urls)
cleanup(urls)
fetch_parallel(urls)

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.