Skip to content

Instantly share code, notes, and snippets.

@abusedmedia
Created February 4, 2016 18:34
Show Gist options
  • Save abusedmedia/9ddb9795c244e004090d to your computer and use it in GitHub Desktop.
Save abusedmedia/9ddb9795c244e004090d to your computer and use it in GitHub Desktop.
Node script for bulk image download, image scraping for the masses
var fs = require('fs'),
request = require('request');
src = require('./dataset.js'); //external array with url of images that need to be downloaded
var download = function(uri, filename, callback){
request.head({url:uri, followAllRedirects: true}, function(err, res, body){
console.log('content-type:', res.headers['content-type']);
console.log('content-length:', res.headers['content-length']);
var content = res.headers['content-type'];
var ext = content.split('/')
console.log(ext);
filename = filename + '.' + ext[1];
var r = request({url:uri, followAllRedirects: true}).pipe(fs.createWriteStream(filename));
r.on('close', callback);
r.on('error', function(err){
console.log(err)
});
});
};
var count = 16;
function doLoad(){
var ob = src.data[count];
console.log('attempt', ob);
download('http:' + ob, 'downloaded/' + count, function(){
count++;
if(count>=src.data.length){
console.log('Done!')
}else{
doLoad();
}
});
}
doLoad();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment