Skip to content

Instantly share code, notes, and snippets.

@sanfx
Created June 14, 2014 15:41
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save sanfx/aed0b65cfa1e5318abb0 to your computer and use it in GitHub Desktop.
Save sanfx/aed0b65cfa1e5318abb0 to your computer and use it in GitHub Desktop.
Download multiple files parallel with multiple threads for each download.
import urllib2
import os
import sys
import time
import threading
urls = ["http://broadcast.lds.org/churchmusic/MP3/1/2/nowords/271.mp3",
"http://s1.fans.ge/mp3/201109/08/John_Legend_So_High_Remix(fans_ge).mp3",
"http://megaboon.com/common/preview/track/786203.mp3"]
url = urls[1]
def downloadFile(url, saveTo=None):
file_name = url.split('/')[-1]
if not saveTo:
saveTo = '/Users/userName/Desktop'
try:
u = urllib2.urlopen(url)
except urllib2.URLError , er:
print("%s for %s failed to download." % (er.reason, os.path.basename(url)))
else:
f = open(os.path.join(saveTo, file_name), 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
print "Downloading: %s Bytes: %s" % (file_name, file_size)
file_size_dl = 0
block_sz = 8192
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
status = status + chr(8)*(len(status)+1)
sys.stdout.write('%s\r' % status)
time.sleep(.2)
sys.stdout.flush()
if file_size_dl == file_size:
print r"Download Completed %s%% for file %s, saved to %s" % (file_size_dl * 100. / file_size, file_name, saveTo,)
f.close()
return
def synchronusDownload():
urls_saveTo = {urls[0]: None, urls[1]: None, urls[2]: None}
for url, saveTo in urls_saveTo.iteritems():
th = threading.Thread(target=downloadFile, args=(url, saveTo), name="%s_Download_Thread" % os.path.basename(url))
th.start()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment