AWS Bucket Object transfer between source bucket and destination bucket From: http://www.sebastianseilund.com/nodejs-async-in-practice

AWS Bucket Transfer

//Prepare S3 access and bucket names
var awssum = require('awssum');
var s3 = new awssum.load('amazon/s3').S3({
    accessKeyId: '...',
    secretAccessKey: '..',
});
var sourceBucket = 'old-bucket';
var destinationBucket = 'new-bucket';
var listObjectsDone = false;

//Set up our queue
var queue = async.queue(function(objectName, callback) {
    //This is the queue's task function
    //It copies objectName from source- to destination bucket
    var options = {
        BucketName: destinationBucket,
        ObjectName: objectName,
        SourceBucket: sourceBucket,
        SourceObject: objectName
    };
    s3.CopyObject(options, function(err) {
        if (err) throw err;
        callback(); //Tell async that this queue item has been processed
    });
}, 20); //Only allow 20 copy requests at a time
//When the queue is emptied we want to check if we're done
queue.drain = function() {
    checkDone();
};

//Define the function that lists objects from the source bucket
function listObjects(marker) {
    var options = {
        BucketName: sourceBucket,
        Marker: marker,
        MaxKeys: 1000
    };
    s3.ListObjects(options, function(err, data) {
        if (err) throw err;
        var result = data.Body.ListBucketResult;
        var contents = _.isArray(result.Contents) ? result.Contents : [result.Contents]; //AWS sends an array if multiple, and a single object if there was only one result
        _.each(contents, function(item) {
            var objectName = item.Key;
            marker = objectName; //Save the marker
            queue.push(objectName); //Push the object to our queue
        });
        if (result.IsTruncated == 'true') {
            //The result is truncated, i.e. we have to list once more from the new marker
            listObjects(marker);
        } else {
            listObjectsDone = true; //Tell our routine that we don't need to wait for more objects from S3 
            checkDone();
        }
    });
}

//This function gets called when a) list didn't return a truncated result (because we were at the end), and b) when the last task of the queue is finished
function checkDone() {
    if (queue.length() == 0 && listObjectsDone) {
        console.log('Tada! All objects have been copied :)');
    }
}

//Start the routine by calling listObjects with null as the marker
listObjects(null);