Skip to content

Instantly share code, notes, and snippets.

@newtriks
Created February 6, 2014 18:35
Show Gist options
  • Star 6 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save newtriks/8849948 to your computer and use it in GitHub Desktop.
Save newtriks/8849948 to your computer and use it in GitHub Desktop.
NodeJS module to download multiple files from Amazon S3
#!/usr/bin/env node
var Q = require('q'),
FS = require('fs'),
PB = require('progress'),
AWS = require('aws-sdk'),
conf = new require('../config')();
AWS.config.update(conf.aws_credentials);
var s3 = new AWS.S3();
function downloadFile(filename) {
var deferred = Q.defer(),
output = conf.inputFolder + filename,
stream = FS.createWriteStream(output),
params = {
Bucket: conf.S3bucket,
Key: filename
};
var bar;
s3.getObject(params)
.on('httpHeaders', function (statusCode, headers, resp) {
var len = parseInt(headers['content-length'], 10);
bar = new PB(' ' + filename + ': [:bar] :percent :etas', {
complete: '=',
incomplete: ' ',
width: 20,
total: len
});
})
.on('httpData', function (chunk) {
stream.write(chunk);
bar.tick(chunk.length);
})
.on('httpDone', function (response) {
if (response.error) {
deferred.reject(response.error);
} else {
deferred.resolve(output);
}
stream.end();
})
.send();
return deferred.promise;
}
module.exports = {
'getFiles': function (files) {
return Q.all(files.map(downloadFile));
}
};
// *********** --------------------- *********** //
// *********** example usage: app.js *********** //
// *********** --------------------- *********** //
// #!/usr/bin/env node
// var Q = require('q'),
// fileAPI = require('./lib/file'),
// filenames = [
// 'files/one.pdf',
// 'files/two.pdf',
// 'files/three.ppt',
// 'files/four.ppt'
// ];
// fileAPI.getFiles(filenames)
// .then(console.log)
// .fail(function (error) {
// console.error('Error: ' + error.statusCode + ' - ' + error.message);
// });
// *********** // example usage: app.js *********** //
// *********** -------------- *********** //
// *********** example output *********** //
// *********** -------------- *********** //
// files/one.pdf: [===================] 100% 0.0s
// files/two.pdf: [===================] 100% 0.0s
// files/three.ppt: [===================] 100% 0.0s
// conversions/four.ppt: [===================] 100% 0.0s
// *********** // example output *********** //
@mgurnani
Copy link

mgurnani commented Jun 7, 2017

Nice piece of code.. Thanks for sharing !

@vinkrish
Copy link

thanks for the gist, do you have any suggestions on how the above code can be used with koajs or expressjs allowing download of multiple files?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment