Skip to content

Instantly share code, notes, and snippets.

@machty
Created June 14, 2012 17:29
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save machty/2931630 to your computer and use it in GitHub Desktop.
Save machty/2931630 to your computer and use it in GitHub Desktop.
DOWNLOAD_CONCURRENCY = 10
# Data Structures
longPollRequests = []
jobs = {}
downloadQueue = async.queue (photo, callback) ->
# Here we define the task handled by this concurrency-limiting queue.
downloadfile = photo.url
host = url.parse(downloadfile).hostname
filename = url.parse(downloadfile).pathname.split("/").pop()
filename = "./public/downloads/#{filename}"
photo.localFilename = filename
console.log "set photo.localFilename to #{photo.localFilename}"
theurl = http.createClient(80, host)
requestUrl = downloadfile
request = theurl.request('GET', requestUrl, {"host": host})
request.end()
request.addListener 'response', (response) ->
downloadfile = fs.createWriteStream(filename, {'flags': 'a'})
#sys.puts("File size " + filename + ": " + response.headers['content-length'] + " bytes.")
response.addListener 'data', (chunk) ->
downloadfile.write(chunk, encoding='binary');
response.addListener "end", ->
downloadfile.end()
sys.puts("File size " + filename + ": " + response.headers['content-length'] + " bytes.")
#callback photo
, DOWNLOAD_CONCURRENCY
downloadPhotos = (job) ->
# Enqueue all photos
downloadQueue.push job.photos, (error, photo) ->
if error
job.statuses.push "ERROR: Finished downloading #{photo.localFilename}"
else
job.statuses.push "Finished downloading #{photo.localFilename}"
if --imagesToDownload == 0
job.statuses.push "Finished downloading all files"
publish job
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment