Skip to content

Instantly share code, notes, and snippets.

@marcuswestin
Created November 28, 2014 03:24
Show Gist options
  • Save marcuswestin/217edb025909b7aa4331 to your computer and use it in GitHub Desktop.
Save marcuswestin/217edb025909b7aa4331 to your computer and use it in GitHub Desktop.
Download a bunch of files in parallel. Depends on `npm install request`
var fs = require('fs'),
request = require('request');
var download = function(uri, filename, callback){
request(uri).pipe(fs.createWriteStream(filename)).on('close', callback);
}
var urls = JSON.parse(fs.readFileSync('./urls.json'))
for (var i=0; i<25; i++) {
next()
}
function next() {
if (!urls.length) {
console.log("DONE!", urls.length, 'files.')
return
}
console.log("Remaining:", urls.length)
var url = urls.pop()
var filename = 'files/'+url.split('/').pop()
if (fs.existsSync(filename)) {
console.log("Already downloaded:", filename)
next()
return
}
console.log("Downloading:", filename, url)
download(url, filename, function(err) {
if (err) {
console.log("ERROR", err)
process.exit(1)
return
}
console.log("Done:", url)
next()
})
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment