Skip to content

Instantly share code, notes, and snippets.

@michaelwittig
Created November 13, 2015 07:59
Show Gist options
  • Save michaelwittig/7394988bca1dfe7e011b to your computer and use it in GitHub Desktop.
Save michaelwittig/7394988bca1dfe7e011b to your computer and use it in GitHub Desktop.
Delete all keys / versions from S3 bucket without API throtteling (flow control)
var MAX_CONCURRENT_API_CALLS = 12;
var BUCKET = "bucket-name";
var AWS = require("aws-sdk");
var s3 = new AWS.S3();
var async = require("async");
var queue = async.queue(function(task, cb) {
console.log(task.action);
if (task.action === "list") {
s3.listObjectVersions({
"Bucket": BUCKET,
"KeyMarker": task.nextKeyMarker,
"VersionIdMarker": task.nextVersionIdMarker
}, function(err, data) {
if (err) {
cb(err);
} else {
var tasks = data.Versions.map(function(version) {
return {"action": "delete", "key": version.Key, "versionId": version.VersionId};
});
queue.push(tasks);
if (data.IsTruncated === true) {
queue.push({"action": "list", "nextKeyMarker": data.NextKeyMarker, "nextVersionIdMarker": data.NextVersionIdMarker});
}
cb();
}
});
} else if (task.action === "delete") {
s3.deleteObject({
"Bucket": BUCKET,
"Key": task.key,
"VersionId": task.versionId
}, cb);
} else {
cb(new Error("unsupported action " + task.action));
}
}, MAX_CONCURRENT_API_CALLS);
queue.push({"action": "list"});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment