async queue download

gistfile1.coffee

ownloadQueue = async.queue (photo, callback) ->
  # Here we define the task handled by this concurrency-limiting queue.
  downloadfile = photo.url
  host = url.parse(downloadfile).hostname
  filename = url.parse(downloadfile).pathname.split("/").pop()
  filename = "./public/downloads/#{filename}"
  photo.localFilename = filename
  console.log "set photo.localFilename to #{photo.localFilename}"
  theurl = http.createClient(80, host)
  requestUrl = downloadfile
  request = theurl.request('GET', requestUrl, {"host": host})
  request.end()

  request.addListener 'response', (response) ->
    downloadfile = fs.createWriteStream(filename, {'flags': 'a'})
    #sys.puts("File size " + filename + ": " + response.headers['content-length'] + " bytes.")
    response.addListener 'data', (chunk) ->
      downloadfile.write(chunk, encoding='binary');
    response.addListener "end", ->
      downloadfile.end()
      callback photo
, DOWNLOAD_CONCURRENCY


# Later...


# Enqueue all photos
downloadQueue.push photos, (photo) ->
  job.statuses.push "Finished downloading #{photo.localFilename}"
  if --imagesToDownload == 0
    job.statuses.push "Finished downloading all files"
  publish job